solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.packagemanager;

import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.lucene.util.SuppressForbidden;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.V2Response;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.BlobRepository;
import org.apache.solr.filestore.DistribPackageStore;
import org.apache.solr.filestore.PackageStoreAPI;
import org.apache.solr.packagemanager.SolrPackage.Manifest;
import org.apache.solr.util.SolrJacksonAnnotationInspector;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.zafarkhaja.semver.Version;
import com.google.common.base.Strings;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
import com.jayway.jsonpath.spi.json.JsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
import com.jayway.jsonpath.spi.mapper.MappingProvider;

public class PackageUtils {

  /**
   * Represents a version which denotes the latest version available at the moment.
   */
  public static String LATEST = "latest";
  
  public static String PACKAGE_PATH = "/api/cluster/package";
  public static String CLUSTER_PLUGINS_PATH = "/api/cluster/plugin";
  public static String REPOSITORIES_ZK_PATH = "/repositories.json";
  public static String CLUSTERPROPS_PATH = "/api/cluster/zk/data/clusterprops.json";
  
 
  public static Configuration jsonPathConfiguration() {
    MappingProvider provider = new JacksonMappingProvider();
    JsonProvider jsonProvider = new JacksonJsonProvider();
    Configuration c = Configuration.builder().jsonProvider(jsonProvider).mappingProvider(provider).options(com.jayway.jsonpath.Option.REQUIRE_PROPERTIES).build();
    return c;
  }

  public static ObjectMapper getMapper() {
    return new ObjectMapper().setAnnotationIntrospector(new SolrJacksonAnnotationInspector());
  }

  /**
   * Uploads a file to the package store / file store of Solr.
   * 
   * @param client A Solr client
   * @param buffer File contents
   * @param name Name of the file as it will appear in the file store (can be hierarchical)
   * @param sig Signature digest (public key should be separately uploaded to ZK)
   */
  public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
      throws SolrServerException, IOException {
    String resource = "/api/cluster/files" + name;
    ModifiableSolrParams params = new ModifiableSolrParams();
    if (sig != null) {
      params.add("sig", sig);
    }
    V2Response rsp = new V2Request.Builder(resource)
        .withMethod(SolrRequest.METHOD.PUT)
        .withPayload(buffer)
        .forceV2(true)
        .withMimeType("application/octet-stream")
        .withParams(params)
        .build()
        .process(client);
    if (!name.equals(rsp.getResponse().get(CommonParams.FILE))) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Mismatch in file uploaded. Uploaded: " +
          rsp.getResponse().get(CommonParams.FILE)+", Original: "+name);
    }
  }

  /**
   * Download JSON from the url and deserialize into klass.
   */
  public static <T> T getJson(HttpClient client, String url, Class<T> klass) {
    try {
      return getMapper().readValue(getJsonStringFromUrl(client, url), klass);
    } catch (IOException e) {
      throw new RuntimeException(e);
    } 
  }

  /**
   * Search through the list of jar files for a given file. Returns string of
   * the file contents or null if file wasn't found. This is suitable for looking
   * for manifest or property files within pre-downloaded jar files.
   * Please note that the first instance of the file found is returned.
   */
  public static String getFileFromJarsAsString(List<Path> jars, String filename) {
    for (Path jarfile: jars) {
      try (ZipFile zipFile = new ZipFile(jarfile.toFile())) {
        ZipEntry entry = zipFile.getEntry(filename);
        if (entry == null) continue;
        return IOUtils.toString(zipFile.getInputStream(entry), "UTF-8");
      } catch (Exception ex) {
        throw new SolrException(ErrorCode.BAD_REQUEST, ex);
      }
    }
    return null;
  }

  /**
   * Returns JSON string from a given URL
   */
  public static String getJsonStringFromUrl(HttpClient client, String url) {
    try {
      HttpResponse resp = client.execute(new HttpGet(url));
      if (resp.getStatusLine().getStatusCode() != 200) {
        throw new SolrException(ErrorCode.NOT_FOUND,
            "Error (code="+resp.getStatusLine().getStatusCode()+") fetching from URL: "+url);
      }
      return IOUtils.toString(resp.getEntity().getContent(), "UTF-8");
    } catch (UnsupportedOperationException | IOException e) {
      throw new RuntimeException(e);
    }
  }

  /**
   * Checks whether a given version satisfies the constraint (defined by a semver expression)
   */
  public static boolean checkVersionConstraint(String ver, String constraint) {
    return Strings.isNullOrEmpty(constraint) || Version.valueOf(ver).satisfies(constraint);
  }

  /**
   * Fetches a manifest file from the File Store / Package Store. A SHA512 check is enforced after fetching.
   */
  public static Manifest fetchManifest(HttpSolrClient solrClient, String solrBaseUrl, String manifestFilePath, String expectedSHA512) throws MalformedURLException, IOException {
    String manifestJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/node/files" + manifestFilePath);
    String calculatedSHA512 = BlobRepository.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes("UTF-8")));
    if (expectedSHA512.equals(calculatedSHA512) == false) {
      throw new SolrException(ErrorCode.UNAUTHORIZED, "The manifest SHA512 doesn't match expected SHA512. Possible unauthorized manipulation. "
          + "Expected: " + expectedSHA512 + ", calculated: " + calculatedSHA512 + ", manifest location: " + manifestFilePath);
    }
    Manifest manifest = getMapper().readValue(manifestJson, Manifest.class);
    return manifest;
  }

  /**
   * Replace a templatized string with parameter substituted string. First applies the overrides, then defaults and then systemParams.
   */
  public static String resolve(String str, Map<String, String> defaults, Map<String, String> overrides, Map<String, String> systemParams) {
    // TODO: Should perhaps use Matchers etc. instead of this clumsy replaceAll().

    if (str == null) return null;
    if (defaults != null) {
      for (String param: defaults.keySet()) {
        str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param));
      }
    }
    for (String param: overrides.keySet()) {
      str = str.replaceAll("\\$\\{"+param+"\\}", overrides.get(param));
    }
    for (String param: systemParams.keySet()) {
      str = str.replaceAll("\\$\\{"+param+"\\}", systemParams.get(param));
    }
    return str;
  }

  /**
   * Compares two versions v1 and v2. Returns negative if v1 isLessThan v2, positive if v1 isGreaterThan v2 and 0 if equal.
   */
  public static int compareVersions(String v1, String v2) {
    return Version.valueOf(v1).compareTo(Version.valueOf(v2));
  }

  public static String BLACK = "\u001B[30m";
  public static String RED = "\u001B[31m";
  public static String GREEN = "\u001B[32m";
  public static String YELLOW = "\u001B[33m";
  public static String BLUE = "\u001B[34m";
  public static String PURPLE = "\u001B[35m";
  public static String CYAN = "\u001B[36m";
  public static String WHITE = "\u001B[37m";

  /**
   * Console print using green color
   */
  public static void printGreen(Object message) {
    PackageUtils.print(PackageUtils.GREEN, message);
  }

  /**
   * Console print using red color
   */
  public static void printRed(Object message) {
    PackageUtils.print(PackageUtils.RED, message);
  }

  public static void print(Object message) {
    print(null, message);
  }

  @SuppressForbidden(reason = "Need to use System.out.println() instead of log4j/slf4j for cleaner output")
  public static void print(String color, Object message) {
    String RESET = "\u001B[0m";

    if (color != null) {
      System.out.println(color + String.valueOf(message) + RESET);
    } else {
      System.out.println(message);
    }
  }

  public static String[] validateCollections(String collections[]) {
    String collectionNameRegex = "^[a-zA-Z0-9_-]*$";
    for (String c: collections) {
      if (c.matches(collectionNameRegex) == false) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid collection name: " + c +
            ". Didn't match the pattern: '"+collectionNameRegex+"'");
      }
    }
    return collections;
  }
  
  public static String getCollectionParamsPath(String collection) {
    return "/api/collections/" + collection + "/config/params";
  }

  public static void uploadKey(byte bytes[], String path, Path home, HttpSolrClient client) throws IOException {
    ByteBuffer buf = ByteBuffer.wrap(bytes);
    PackageStoreAPI.MetaData meta = PackageStoreAPI._createJsonMetaData(buf, null);
    DistribPackageStore._persistToFile(home, path, buf, ByteBuffer.wrap(Utils.toJSON(meta)));
  }

}