utils/src/main/java/com/cloud/utils/UriUtils.java

//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//

package com.cloud.utils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.Inet6Address;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.function.Predicate;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import org.apache.cloudstack.utils.security.ParserUtils;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.util.URIUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.message.BasicNameValuePair;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.HttpClientCloudStackUserAgent;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;

public class UriUtils {

    protected static Logger LOGGER = LogManager.getLogger(UriUtils.class);
    public static final String USER_AGENT = "User-Agent";

    public static String formNfsUri(String host, String path) {
        try {
            URI uri = new URI("nfs", host, path, null);
            return uri.toString();
        } catch (URISyntaxException e) {
            throw new CloudRuntimeException("Unable to form nfs URI: " + host + " - " + path);
        }
    }

    public static String formIscsiUri(String host, String iqn, Integer lun) {
        try {
            String path = iqn;
            if (lun != null) {
                path += "/" + lun.toString();
            }
            URI uri = new URI("iscsi", host, path, null);
            return uri.toString();
        } catch (URISyntaxException e) {
            throw new CloudRuntimeException("Unable to form iscsi URI: " + host + " - " + iqn + " - " + lun);
        }
    }

    public static String formFileUri(String path) {
        File file = new File(path);

        return file.toURI().toString();
    }

    // a simple URI component helper (Note: it does not deal with URI paramemeter area)
    public static String encodeURIComponent(String url) {
        int schemeTail = url.indexOf("://");

        int pathStart = 0;
        if (schemeTail > 0)
            pathStart = url.indexOf('/', schemeTail + 3);
        else
            pathStart = url.indexOf('/');

        if (pathStart > 0) {
            String[] tokens = url.substring(pathStart + 1).split("/");
            StringBuilder sb = new StringBuilder(url.substring(0, pathStart));
            for (String token : tokens) {
                sb.append("/").append(URLEncoder.encode(token));
            }

            return sb.toString();
        }

        // no need to do URL component encoding
        return url;
    }

    public static String getCifsUriParametersProblems(URI uri) {
        if (!UriUtils.hostAndPathPresent(uri)) {
            String errMsg = "cifs URI missing host and/or path. Make sure it's of the format cifs://hostname/path";
            LOGGER.warn(errMsg);
            return errMsg;
        }
        return null;
    }

    public static boolean hostAndPathPresent(URI uri) {
        return !(uri.getHost() == null || uri.getHost().trim().isEmpty() || uri.getPath() == null || uri.getPath().trim().isEmpty());
    }

    public static boolean cifsCredentialsPresent(URI uri) {
        List<NameValuePair> args = URLEncodedUtils.parse(uri, "UTF-8");
        boolean foundUser = false;
        boolean foundPswd = false;
        for (NameValuePair nvp : args) {
            String name = nvp.getName();
            if (name.equals("user")) {
                foundUser = true;
                LOGGER.debug("foundUser is" + foundUser);
            } else if (name.equals("password")) {
                foundPswd = true;
                LOGGER.debug("foundPswd is" + foundPswd);
            }
        }
        return (foundUser && foundPswd);
    }

    public static String getUpdateUri(String url, boolean encrypt) {
        String updatedPath = null;
        try {
            String query = URIUtil.getQuery(url);
            URIBuilder builder = new URIBuilder(url);
            builder.removeQuery();

            StringBuilder updatedQuery = new StringBuilder();
            List<NameValuePair> queryParams = getUserDetails(query);
            ListIterator<NameValuePair> iterator = queryParams.listIterator();
            while (iterator.hasNext()) {
                NameValuePair param = iterator.next();
                String value = null;
                if ("password".equalsIgnoreCase(param.getName()) &&
                        param.getValue() != null) {
                    value = encrypt ? DBEncryptionUtil.encrypt(param.getValue()) : DBEncryptionUtil.decrypt(param.getValue());
                } else {
                    value = param.getValue();
                }

                if (updatedQuery.length() == 0) {
                    updatedQuery.append(param.getName()).append('=')
                            .append(value);
                } else {
                    updatedQuery.append('&').append(param.getName())
                            .append('=').append(value);
                }
            }

            String schemeAndHost = "";
            URI newUri = builder.build();
            if (newUri.getScheme() != null) {
                schemeAndHost = newUri.getScheme() + "://" + newUri.getHost();
            }

            updatedPath = schemeAndHost + newUri.getPath() + "?" + updatedQuery;
        } catch (URISyntaxException e) {
            throw new CloudRuntimeException("Couldn't generate an updated uri. " + e.getMessage());
        }

        return updatedPath;
    }

    private static List<NameValuePair> getUserDetails(String query) {
        List<NameValuePair> details = new ArrayList<NameValuePair>();
        if (query != null && !query.isEmpty()) {
            StringTokenizer allParams = new StringTokenizer(query, "&");
            while (allParams.hasMoreTokens()) {
                String param = allParams.nextToken();
                details.add(new BasicNameValuePair(param.substring(0, param.indexOf("=")),
                        param.substring(param.indexOf("=") + 1)));
            }
        }

        return details;
    }

    // Get the size of a file from URL response header.
    public static long getRemoteSize(String url, Boolean followRedirect) {
        long remoteSize = 0L;
        final String[] methods = new String[]{"HEAD", "GET"};
        IllegalArgumentException exception = null;
        // Attempting first a HEAD request to avoid downloading the whole file. If
        // it fails (for example with S3 presigned URL), fallback on a standard GET
        // request.
        for (String method : methods) {
            HttpURLConnection httpConn = null;
            try {
                URI uri = new URI(url);
                httpConn = (HttpURLConnection)uri.toURL().openConnection();
                httpConn.setRequestMethod(method);
                httpConn.setRequestProperty(USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
                httpConn.setConnectTimeout(2000);
                httpConn.setReadTimeout(5000);
                httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));
                String contentLength = httpConn.getHeaderField("content-length");
                if (contentLength != null) {
                    remoteSize = Long.parseLong(contentLength);
                } else if (method.equals("GET") && httpConn.getResponseCode() < 300) {
                    // Calculate the content size based on the input stream content
                    byte[] buf = new byte[1024];
                    int length;
                    while ((length = httpConn.getInputStream().read(buf, 0, buf.length)) != -1) {
                        remoteSize += length;
                    }
                }
                return remoteSize;
            } catch (URISyntaxException e) {
                throw new IllegalArgumentException("Invalid URL " + url);
            } catch (IOException e) {
                exception = new IllegalArgumentException("Unable to establish connection with URL " + url);
            } finally {
                if (httpConn != null) {
                    httpConn.disconnect();
                }
            }
        }
        if (exception != null) {
            throw exception;
        }
        return 0L;
    }

    public static Pair<String, Integer> validateUrl(String url) throws IllegalArgumentException {
        return validateUrl(null, url);
    }

    public static Pair<String, Integer> validateUrl(String format, String url) throws IllegalArgumentException {
        return validateUrl(format, url, false, false);
    }

    /**
     * Verifies whether the provided URL is valid.
     * @param skipHostCheck if false, this function will verify whether the provided URL is resolvable, if it is a legal address and if it does not use IPv6 (configured by `skipIpv6Check`). If any of these conditions are false, an exception will be thrown.
     * @param skipIpv6Check if false, this function will verify whether the host uses IPv6 and, if it does, an exception will be thrown. This check is also skipped if `skipHostCheck` is true.
     * @return a pair containing the host and the corresponding port.
     * @throws IllegalArgumentException if the provided URL is invalid.
     */
    public static Pair<String, Integer> validateUrl(String format, String url, boolean skipHostCheck, boolean skipIpv6Check) throws IllegalArgumentException {
        try {
            URI uri = new URI(url);
            if ((uri.getScheme() == null) ||
                    (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") && !uri.getScheme().equalsIgnoreCase("file"))) {
                throw new IllegalArgumentException("Unsupported scheme for url: " + url);
            }

            int port = uri.getPort();
            if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) {
                port = 443;
            } else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) {
                port = 80;
            }

            String host = uri.getHost();
            if (!skipHostCheck) {
                checkHost(host, skipIpv6Check);
            }

            // verify format
            if (format != null) {
                String uripath = uri.getPath();
                checkFormat(format, uripath);
            }
            return new Pair<String, Integer>(host, port);
        } catch (URISyntaxException use) {
            throw new IllegalArgumentException("Invalid URL: " + url);
        }
    }

    /**
     * Validates a URL extracted from inside a metalink file.
     * Only {@code http} and {@code https} schemes are permitted.
     * Loopback, link-local, any-local, multicast, and site-local (RFC-1918)
     * addresses are all blocked.
     *
     * @param url the inner URL to validate
     * @throws IllegalArgumentException if the URL is not safe to fetch
     */
    public static void validateMetalinkInnerUrl(String url) throws IllegalArgumentException {
        validateMetalinkInnerUrl(url, Collections.emptyList());
    }

    /**
     * Validates a URL extracted from inside a metalink file, with an optional
     * admin-configured CIDR allowlist that may permit private-range addresses.
     * Loopback, link-local, any-local, and multicast addresses are
     * always blocked regardless of the allowlist.
     * Site-local (RFC-1918) addresses can be selectively permitted by
     * supplying matching CIDR strings (e.g. {@code "10.0.0.0/8"}).
     *
     * @param url          the inner URL to validate
     * @param allowedCidrs CIDR strings whose addresses are allowed even when
     *                     site-local; an empty list blocks all site-local addresses
     * @throws IllegalArgumentException if the URL is not safe to fetch
     */
    public static void validateMetalinkInnerUrl(String url, List<String> allowedCidrs)
            throws IllegalArgumentException {
        if (url == null || url.isBlank()) {
            throw new IllegalArgumentException("Empty URL in metalink file");
        }
        try {
            URI uri = new URI(url);
            String scheme = uri.getScheme();
            if (scheme == null
                    || (!scheme.equalsIgnoreCase("http") && !scheme.equalsIgnoreCase("https")
                            && !scheme.equalsIgnoreCase("nfs"))) {
                throw new IllegalArgumentException(
                        String.format("Metalink inner URL scheme not allowed: '%s'. Only http, https and nfs are permitted.", scheme));
            }
            String host = uri.getHost();
            if (host == null || host.isBlank()) {
                throw new IllegalArgumentException(String.format("No host in metalink inner URL: %s", url));
            }
            try {
                InetAddress addr = InetAddress.getByName(host);
                checkHost(addr, false);
                // Site-local (RFC-1918) blocked by default; overridable via allowlist.
                if (addr.isSiteLocalAddress() && !isAddressInAllowedCidrs(host, addr, allowedCidrs)) {
                    throw new IllegalArgumentException(
                            String.format("Metalink inner URL resolves to a private/site-local address: %s. "
                                    + "Configure direct.download.metalink.allowed.hosts.and.cidrs to permit specific ranges.", host));
                }
            } catch (UnknownHostException e) {
                throw new IllegalArgumentException(String.format("Unable to resolve metalink inner URL host: %s", host));
            }
        } catch (URISyntaxException e) {
            throw new IllegalArgumentException(String.format("Invalid metalink inner URL: %s", url));
        }
    }

    static boolean isAddressInAllowedCidrs(InetAddress addr, List<String> entries) {
        return isAddressInAllowedCidrs(addr.getHostAddress(), addr, entries);
    }

    /**
     * Returns true if the given host/address is permitted by any entry in the list.
     * Entries may be a CIDR range 10.0.0.0/8}, an exact hostname or IP
     * (storage.corp.com), or a wildcard domain suffix (*.mylocal.net).
     * Wildcard matching is a hostname-suffix check against host; CIDR and
     * exact-hostname entries resolve to an IP and compare. Malformed entries are logged and skipped.
     */
    static boolean isAddressInAllowedCidrs(String host, InetAddress addr, List<String> entries) {
        if (entries == null || entries.isEmpty()) {
            return false;
        }
        byte[] addrBytes = addr.getAddress();
        for (String entry : entries) {
            String trimmed = entry.trim();
            if (trimmed.startsWith("*.")) {
                String suffix = trimmed.substring(1); // e.g. ".mylocal.net"
                if (host != null && host.toLowerCase().endsWith(suffix.toLowerCase())) {
                    return true;
                }
            } else {
                try {
                    if (trimmed.contains("/")) {
                        String[] parts = trimmed.split("/");
                        if (parts.length != 2) {
                            LOGGER.warn("Ignoring malformed CIDR in metalink allowlist: " + trimmed);
                            continue;
                        }
                        InetAddress cidrAddr = InetAddress.getByName(parts[0]);
                        int prefixLen = Integer.parseInt(parts[1].trim());
                        byte[] cidrBytes = cidrAddr.getAddress();
                        if (cidrBytes.length != addrBytes.length) {
                            continue;
                        }
                        if (isMatchingPrefix(addrBytes, cidrBytes, prefixLen)) {
                            return true;
                        }
                    } else {
                        if (InetAddress.getByName(trimmed).equals(addr)) {
                            return true;
                        }
                    }
                } catch (UnknownHostException | NumberFormatException e) {
                    LOGGER.warn("Ignoring invalid entry in metalink allowlist: " + trimmed + " — " + e.getMessage());
                }
            }
        }
        return false;
    }

    /** Returns true if {@code addr} matches {@code cidr} for the given prefix length. */
    private static boolean isMatchingPrefix(byte[] addr, byte[] cidr, int prefixLen) {
        int fullBytes = prefixLen / 8;
        int remainingBits = prefixLen % 8;
        for (int i = 0; i < fullBytes; i++) {
            if (addr[i] != cidr[i]) return false;
        }
        if (remainingBits > 0 && fullBytes < addr.length) {
            int mask = 0xFF << (8 - remainingBits);
            return (addr[fullBytes] & mask) == (cidr[fullBytes] & mask);
        }
        return true;
    }

    /**
     * Verifies whether the provided host is valid. Throws an `IllegalArgumentException` if:
     * <ul>
     *     <li>The host is not resolvable;</li>
     *     <li>The host address is illegal (any local, link local, loopback or multicast address);</li>
     *     <li>The host uses IPv6. This check is skipped if `skipIv6Check` is set to true.</li>
     * </ul>
     */
    private static void checkHost(String host, boolean skipIpv6Check) {
        try {
            checkHost(InetAddress.getByName(host), skipIpv6Check);
        } catch (UnknownHostException uhe) {
            throw new IllegalArgumentException(String.format("Unable to resolve %s.", host));
        }
    }

    private static void checkHost(InetAddress addr, boolean skipIpv6Check) {
        if (addr.isAnyLocalAddress() || addr.isLinkLocalAddress() || addr.isLoopbackAddress() || addr.isMulticastAddress()) {
            throw new IllegalArgumentException("Illegal host specified in URL.");
        }
        if (!skipIpv6Check && addr instanceof Inet6Address) {
            throw new IllegalArgumentException(String.format("IPv6 addresses are not supported (%s).", addr.getHostAddress()));
        }
    }

    /**
     * Add element to priority list examining node attributes: priority (for urls) and type (for checksums)
     */
    protected static void addPriorityListElementExaminingNode(String tagName, Node node, List<Pair<String, Integer>> priorityList) {
        Integer priority = Integer.MAX_VALUE;
        String first = node.getTextContent();
        if (node.hasAttributes()) {
            NamedNodeMap attributes = node.getAttributes();
            for (int k=0; k<attributes.getLength(); k++) {
                Node attr = attributes.item(k);
                if (tagName.equals("url") && attr.getNodeName().equals("priority")) {
                    String prio = attr.getNodeValue().replace("\"", "");
                    priority = Integer.parseInt(prio);
                    break;
                } else if (tagName.equals("hash") && attr.getNodeName().equals("type")) {
                    first = "{" + attr.getNodeValue() + "}" + first;
                    break;
                }
            }
        }
        priorityList.add(new Pair<>(first, priority));
    }

    /**
     * Return the list of first elements on the list of pairs
     */
    protected static List<String> getListOfFirstElements(List<Pair<String, Integer>> priorityList) {
        List<String> values = new ArrayList<>();
        for (Pair<String, Integer> pair : priorityList) {
            values.add(pair.first());
        }
        return values;
    }

    /**
     * Return HttpClient with connection timeout
     */
    private static HttpClient getHttpClient() {
        MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
        s_httpClientManager.getParams().setConnectionTimeout(5000);
        return new HttpClient(s_httpClientManager);
    }

    /**
     * Retrieve values from XML documents ordered by ascending priority for each tag name
     */
    public static Map<String, List<String>> getMultipleValuesFromXML(InputStream is, String[] tagNames) {
        Map<String, List<String>> returnValues = new HashMap<String, List<String>>();
        try {
            DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory();
            DocumentBuilder docBuilder = factory.newDocumentBuilder();
            Document doc = docBuilder.parse(is);
            Element rootElement = doc.getDocumentElement();
            for (int i = 0; i < tagNames.length; i++) {
                NodeList targetNodes = rootElement.getElementsByTagName(tagNames[i]);
                if (targetNodes.getLength() <= 0) {
                    LOGGER.error("no " + tagNames[i] + " tag in XML response...");
                } else {
                    List<Pair<String, Integer>> priorityList = new ArrayList<>();
                    for (int j = 0; j < targetNodes.getLength(); j++) {
                        Node node = targetNodes.item(j);
                        addPriorityListElementExaminingNode(tagNames[i], node, priorityList);
                    }
                    priorityList.sort(Comparator.comparing(x -> x.second()));
                    returnValues.put(tagNames[i], getListOfFirstElements(priorityList));
                }
            }
        } catch (Exception ex) {
            LOGGER.error(ex);
        }
        return returnValues;
    }

    /**
     * Get list of urls on metalink ordered by ascending priority (for those which priority tag is not defined, highest priority value is assumed)
     */
    public static List<String> getMetalinkUrls(String metalinkUrl) {
        HttpClient httpClient = getHttpClient();
        GetMethod getMethod = new GetMethod(metalinkUrl);
        List<String> urls = new ArrayList<>();
        int status;
        try {
            status = httpClient.executeMethod(getMethod);
        } catch (IOException e) {
            LOGGER.error("Error retrieving urls form metalink: " + metalinkUrl);
            getMethod.releaseConnection();
            return null;
        }
        try {
            InputStream is = getMethod.getResponseBodyAsStream();
            if (status == HttpStatus.SC_OK) {
                Map<String, List<String>> metalinkUrlsMap = getMultipleValuesFromXML(is, new String[] {"url"});
                if (metalinkUrlsMap.containsKey("url")) {
                    List<String> metalinkUrls = metalinkUrlsMap.get("url");
                    urls.addAll(metalinkUrls);
                }
            }
        } catch (IOException e) {
            LOGGER.warn(e.getMessage());
        } finally {
            getMethod.releaseConnection();
        }
        return urls;
    }

    public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz", "xz");

    public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {
        final ImmutableSet.Builder<String> builder = ImmutableSet.builder();

        for (String baseExtension : baseExtensions) {
            builder.add("." + baseExtension);
            for (String format : COMPRESSION_FORMATS) {
                builder.add("." + baseExtension + "." + format);
            }
        }

        if (metalink) {
            builder.add(".metalink");
        }

        return builder.build();
    }

    private final static Map<String, Set<String>> SUPPORTED_EXTENSIONS_BY_FORMAT =
            ImmutableMap.<String, Set<String>>builder()
                        .put("vhd", buildExtensionSet(false, "vhd"))
                        .put("vhdx", buildExtensionSet(false, "vhdx"))
                        .put("qcow2", buildExtensionSet(true, "qcow2", "img"))
                        .put("ova", buildExtensionSet(true, "ova"))
                        .put("tar", buildExtensionSet(false, "tar"))
                        .put("raw", buildExtensionSet(false, "img", "raw"))
                        .put("vmdk", buildExtensionSet(false, "vmdk"))
                        .put("iso", buildExtensionSet(true, "iso"))
            .build();

    public final static Set<String> getSupportedExtensions(String format) {
        return SUPPORTED_EXTENSIONS_BY_FORMAT.get(format);
    }

    // verify if a URI path is compliance with the file format given
    private static void checkFormat(String format, String uripath) {
        final String lowerCaseUri = uripath.toLowerCase();

        final boolean unknownExtensionForFormat = SUPPORTED_EXTENSIONS_BY_FORMAT.get(format.toLowerCase())
                                                                                .stream()
                                                                                .noneMatch(lowerCaseUri::endsWith);

        if (unknownExtensionForFormat) {
            final Predicate<Set<String>> uriMatchesAnyExtension =
                    supportedExtensions -> supportedExtensions.stream()
                                                              .anyMatch(lowerCaseUri::endsWith);

            boolean unknownExtension = SUPPORTED_EXTENSIONS_BY_FORMAT.values()
                                                                     .stream()
                                                                     .noneMatch(uriMatchesAnyExtension);

            if (unknownExtension) {
                throw new IllegalArgumentException("Please specify a valid " + format.toLowerCase());
            }

            throw new IllegalArgumentException("Please specify a valid URL. "
                                                       + "URL:" + uripath + " is an invalid for the format " + format.toLowerCase());
        }
    }

    public static InputStream getInputStreamFromUrl(String url, String user, String password) {

        try {
            Pair<String, Integer> hostAndPort = validateUrl(url);
            HttpClient httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
            if ((user != null) && (password != null)) {
                httpclient.getParams().setAuthenticationPreemptive(true);
                Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
                httpclient.getState().setCredentials(
                        new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
                LOGGER.info("Added username={} along with password for host {}:{}"
                    , user
                    , hostAndPort.first()
                    , hostAndPort.second());
            }
            // Execute the method.
            GetMethod method = new GetMethod(url);
            int statusCode = httpclient.executeMethod(method);

            if (statusCode != HttpStatus.SC_OK) {
                LOGGER.error("Failed to read from URL: " + url);
                return null;
            }

            return method.getResponseBodyAsStream();
        } catch (Exception ex) {
            LOGGER.error("Failed to read from URL: " + url);
            return null;
        }
    }

    /**
     * Expands a given vlan URI to a list of vlan IDs
     * @param vlanAuthority the URI part without the vlan:// scheme
     * @return returns list of vlan integer ids
     */
    public static List<Integer> expandVlanUri(final String vlanAuthority) {
        final List<Integer> expandedVlans = new ArrayList<>();
        if (StringUtils.isEmpty(vlanAuthority)) {
            return expandedVlans;
        }
        for (final String vlanPart: vlanAuthority.split(",")) {
            if (StringUtils.isEmpty(vlanPart)) {
                continue;
            }
            final String[] range = vlanPart.split("-");
            if (range.length == 2) {
                Integer start = NumbersUtil.parseInt(range[0], -1);
                Integer end = NumbersUtil.parseInt(range[1], -1);
                if (start <= end && end > -1 && start > -1) {
                    while (start <= end) {
                        expandedVlans.add(start++);
                    }
                }
            } else {
                final Integer value = NumbersUtil.parseInt(range[0], -1);
                if (value > -1) {
                    expandedVlans.add(value);
                }
            }
        }
        return expandedVlans;
    }

    /**
     * Checks if given vlan URI authorities overlap
     * @param vlanRange1
     * @param vlanRange2
     * @return true if they overlap
     */
    public static boolean checkVlanUriOverlap(final String vlanRange1, final String vlanRange2) {
        final List<Integer> vlans1 = expandVlanUri(vlanRange1);
        final List<Integer> vlans2 = expandVlanUri(vlanRange2);
        if (vlans1 == null || vlans2 == null) {
            return true;
        }
        return !Collections.disjoint(vlans1, vlans2);
    }

    public static List<Integer> expandPvlanUri(String pvlanRange) {
        final List<Integer> expandedVlans = new ArrayList<>();
        if (StringUtils.isEmpty(pvlanRange)) {
            return expandedVlans;
        }
        String[] parts = pvlanRange.split("-\\w");
        expandedVlans.add(Integer.parseInt(parts[0]));
        expandedVlans.add(Integer.parseInt(parts[1]));
        return expandedVlans;
    }

    public static class UriInfo {
        String scheme;
        String storageHost;
        String storagePath;
        String userInfo;
        int port = -1;

        public UriInfo() {
        }

        public UriInfo(String scheme, String storageHost, String storagePath, String userInfo, int port) {
            this.scheme = scheme;
            this.storageHost = storageHost;
            this.storagePath = storagePath;
            this.userInfo = userInfo;
            this.port = port;
        }

        public String getScheme() {
            return scheme;
        }

        public String getStorageHost() {
            return storageHost;
        }

        public String getStoragePath() {
            return storagePath;
        }

        public String getUserInfo() {
            return userInfo;
        }

        public int getPort() {
            return port;
        }

        @Override
        public String toString() {
            return String.format("%s://%s%s%s%s", scheme,
                    userInfo == null ? "" : userInfo + "@",
                    storageHost,
                    port == -1 ? "" : ":" + port,
                    storagePath == null ? "" : storagePath);
        }
    }

    public static UriInfo getUriInfo(String url) {
        try {
            if (url == null) {
                return new UriInfo();
            }
            if (url.startsWith("rbd://")) {
                return getRbdUrlInfo(url);
            }
            URI uri = new URI(UriUtils.encodeURIComponent(url));
            return new UriInfo(uri.getScheme(), uri.getHost(), uri.getPath(), uri.getUserInfo(), uri.getPort());
        } catch (URISyntaxException e) {
            throw new CloudRuntimeException(url + " is not a valid uri");
        }
    }

    private static UriInfo getRbdUrlInfo(String url) {
        if (url == null || !url.toLowerCase().startsWith("rbd://")) {
            throw new CloudRuntimeException("RBD URL must start with \"rbd://\"");
        }
        String schema = StringUtils.substring(url, 0, 6);
        url = StringUtils.substring(url, 6, url.length());
        int firstAt = StringUtils.indexOf(url, "@");
        String credentials = (firstAt == -1) ? null : StringUtils.substring(url, 0, firstAt);
        String hostInfo = (firstAt == -1) ? url : StringUtils.substring(url, firstAt + 1, url.length());

        int firstSlash = StringUtils.indexOf(hostInfo, "/");
        int lastColon = StringUtils.lastIndexOf(hostInfo,":");
        int lastSquareBracket = StringUtils.lastIndexOf(hostInfo,"]");
        int endOfHost = lastColon == -1 ? (firstSlash > 0 ? firstSlash : hostInfo.length() + 1) :
                (lastSquareBracket > lastColon ? lastSquareBracket + 1 : lastColon);
        String storageHosts = StringUtils.substring(hostInfo, 0, endOfHost);
        String firstHost = storageHosts.split(",")[0];
        String strAfterHosts = StringUtils.substring(hostInfo, endOfHost);
        try {
            URI uri = new URI(UriUtils.encodeURIComponent(schema + firstHost + strAfterHosts));
            if (credentials != null) {
                credentials = credentials.replace("+", "-");
                credentials = credentials.replace("/", "_");
            }
            return new UriInfo(uri.getScheme(), storageHosts, uri.getPath(), credentials, uri.getPort());
        } catch (URISyntaxException e) {
            throw new CloudRuntimeException(url + " is not a valid uri for RBD");
        }
    }

    public static boolean isUrlForCompressedFile(String url) {
        return UriUtils.COMPRESSION_FORMATS.stream().anyMatch(f -> url.toLowerCase().endsWith(f));
    }
}