dev/tasks/linux-packages/Rakefile

# -*- ruby -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

require "English"

require_relative "../../release/binary-task"
require_relative "helper"

packages = [
  "apache-arrow",
  "apache-arrow-apt-source",
  "apache-arrow-release",
]


namespace :apt do
  desc "Build deb packages"
  task :build do
    packages.each do |package|
      cd(package) do
        ruby("-S", "rake", "apt:build")
      end
    end
  end
end

namespace :yum do
  desc "Build RPM packages"
  task :build do
    packages.each do |package|
      cd(package) do
        ruby("-S", "rake", "yum:build")
      end
    end
  end
end

namespace :version do
  desc "Update versions"
  task :update do
    packages.each do |package|
      cd(package) do
        ruby("-S", "rake", "version:update")
      end
    end
  end
end

namespace :docker do
  desc "Pull built images"
  task :pull do
    packages.each do |package|
      cd(package) do
        ruby("-S", "rake", "docker:pull")
      end
    end
  end

  desc "Push built images"
  task :push do
    packages.each do |package|
      cd(package) do
        ruby("-S", "rake", "docker:push")
      end
    end
  end
end


class LocalBinaryTask < BinaryTask
  include Helper::ApacheArrow

  def initialize(packages)
    @packages = packages
    super()
  end

  def define
    define_apt_test_task
    define_yum_test_task
  end

  private
  def latest_commit_time(git_directory)
    cd(git_directory) do
      return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc
    end
  end

  def version
    @version ||= detect_version(detect_release_time)
  end

  def resolve_docker_image(target)
    image = ""
    case target
    when /-(?:arm64|aarch64)\z/
      target = $PREMATCH
      image << "arm64v8/"
    end
    image << target.gsub(/-/, ":")
  end

  def verify(target)
    verify_command_line = [
      "docker",
      "run",
      "--rm",
      "--log-driver", "none",
      "--volume", "#{File.expand_path(arrow_source_dir)}:/arrow:delegated",
    ]
    if $stdin.tty?
      verify_command_line << "--interactive"
      verify_command_line << "--tty"
    else
      verify_command_line.concat(["--attach", "STDOUT"])
      verify_command_line.concat(["--attach", "STDERR"])
    end
    verify_command_line << resolve_docker_image(target)
    case target
    when /\Adebian-/, /\Aubuntu-/
      verify_command_line << "/arrow/dev/release/verify-apt.sh"
    else
      verify_command_line << "/arrow/dev/release/verify-yum.sh"
    end
    verify_command_line << version
    verify_command_line << "local"
    sh(*verify_command_line)
  end

  def apt_test_targets
    targets = (ENV["APT_TARGETS"] || "").split(",")
    targets = apt_test_targets_default if targets.empty?
    targets
  end

  def apt_test_targets_default
    # Disable arm64 targets by default for now
    # because they require some setups on host.
    [
      "debian-buster",
      # "debian-buster-arm64",
      "debian-bullseye",
      # "debian-bullseye-arm64",
      "ubuntu-xenial",
      # "ubuntu-xenial-arm64",
      "ubuntu-bionic",
      # "ubuntu-bionic-arm64",
      "ubuntu-focal",
      # "ubuntu-focal-arm64",
      "ubuntu-groovy",
      # "ubuntu-groovy-arm64",
    ]
  end

  def define_apt_test_task
    namespace :apt do
      desc "Test deb packages"
      task :test do
        repositories_dir = "apt/repositories"
        rm_rf(repositories_dir)
        @packages.each do |package|
          package_repositories = "#{package}/apt/repositories"
          next unless File.exist?(package_repositories)
          sh("rsync", "-a", "#{package_repositories}/", repositories_dir)
        end
        Dir.glob("#{repositories_dir}/ubuntu/pool/*") do |code_name_dir|
          universe_dir = "#{code_name_dir}/universe"
          next unless File.exist?(universe_dir)
          mv(universe_dir, "#{code_name_dir}/main")
        end
        apt_update(repositories_dir)
        apt_test_targets.each do |target|
          verify(target)
        end
      end
    end
  end

  def yum_test_targets
    targets = (ENV["YUM_TARGETS"] || "").split(",")
    targets = yum_test_targets_default if targets.empty?
    targets
  end

  def yum_test_targets_default
    # Disable aarch64 targets by default for now
    # because they require some setups on host.
    [
      "centos-7",
      "centos-8",
      # "centos-8-aarch64",
    ]
  end

  def define_yum_test_task
    namespace :yum do
      desc "Test RPM packages"
      task :test do
        repositories_dir = "yum/repositories"
        rm_rf(repositories_dir)
        @packages.each do |package|
          package_repositories = "#{package}/yum/repositories"
          next unless File.exist?(package_repositories)
          sh("rsync", "-a", "#{package_repositories}/", repositories_dir)
        end
        rpm_sign(repositories_dir)
        yum_update(repositories_dir)
        yum_test_targets.each do |target|
          verify(target)
        end
      end
    end
  end
end

local_binary_task = LocalBinaryTask.new(packages)
local_binary_task.define