Revert "Use git-ls-files as the source of file paths for license checks. (#15200)"
This reverts commit 5227fd72bd0c65224eb643bc2dfbaf77dd90f045.
diff --git a/.hgignore b/.hgignore
new file mode 100644
index 0000000..1e17ec7
--- /dev/null
+++ b/.hgignore
@@ -0,0 +1,4 @@
+syntax: glob
+*/build/*
+*.class
+
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java
deleted file mode 100644
index d00a28a..0000000
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.gradle.plugins.gitinfo;
-
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import javax.inject.Inject;
-import org.gradle.api.GradleException;
-import org.gradle.api.provider.ValueSource;
-import org.gradle.process.ExecOperations;
-import org.slf4j.LoggerFactory;
-
-/** Returns a list of versioned and non-versioned (but not ignored) git files. */
-public abstract class GitFileListValueSource
- implements ValueSource<List<String>, GitValueSourceParameters> {
-
- @Inject
- public abstract ExecOperations getExecOps();
-
- @Override
- public List<String> obtain() {
- try (var baos = new ByteArrayOutputStream();
- var out = new BufferedOutputStream(baos)) {
- var result =
- getExecOps()
- .exec(
- spec -> {
- spec.setStandardOutput(out);
- spec.setErrorOutput(out);
- spec.setIgnoreExitValue(true);
-
- getParameters().configure(spec);
-
- spec.setExecutable(getParameters().getGitExec().get());
- spec.args(
- "ls-files",
- // show cached
- "-c",
- // show others
- "-o",
- // apply .gitignore exclusions
- "--exclude-standard",
- // don't quote paths, 0-terminate strings.
- "-z");
- });
- out.flush();
-
- // Assume the output is UTF-8, even if it has 0 eols.
- String gitOutput = baos.toString(StandardCharsets.UTF_8);
-
- if (result.getExitValue() != 0) {
- // Something went wrong. Assume this isn't a git checkout and
- // return an empty list of files.
- LoggerFactory.getLogger(getClass())
- .warn(
- "Failed executing git (exit status: {}). An empty list of versioned files will be used (run 'git init'?).",
- result.getExitValue());
-
- return List.of();
- }
-
- return List.of(gitOutput.split("\u0000"));
- } catch (IOException e) {
- throw new GradleException("Errors calling git to fetch local repository status.", e);
- }
- }
-}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
index 84832c9..f6b3df2 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java
@@ -17,7 +17,6 @@
package org.apache.lucene.gradle.plugins.gitinfo;
import org.gradle.api.file.FileSystemLocation;
-import org.gradle.api.provider.ListProperty;
import org.gradle.api.provider.MapProperty;
import org.gradle.api.provider.Property;
@@ -33,10 +32,4 @@
* @return Return the location of {@code .git} directory (or file, if worktrees are used).
*/
public abstract Property<FileSystemLocation> getDotGitDir();
-
- /**
- * @return Return a set of all versioned and non-versioned files (that are not ignored by {@code
- * .gitignore}).
- */
- public abstract ListProperty<String> getAllNonIgnoredProjectFiles();
}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java
index e38a8d7..a7f6459 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java
@@ -19,68 +19,55 @@
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;
-import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;
-import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.file.Directory;
-import org.gradle.api.file.FileSystemLocation;
-import org.gradle.api.provider.Property;
-import org.gradle.api.provider.ValueSourceSpec;
public class GitInfoPlugin extends LuceneGradlePlugin {
@Override
public void apply(Project project) {
applicableToRootProjectOnly(project);
+ var gitInfoProvider =
+ project
+ .getProviders()
+ .of(
+ GitInfoValueSource.class,
+ spec -> {
+ spec.getParameters().getRootProjectDir().set(project.getProjectDir());
+ });
+
var gitInfoExtension =
project.getExtensions().create(GitInfoExtension.NAME, GitInfoExtension.class);
- var providers = project.getProviders();
- Property<FileSystemLocation> dotGitDir = gitInfoExtension.getDotGitDir();
- dotGitDir
+ gitInfoExtension.getGitInfo().value(gitInfoProvider).finalizeValueOnRead();
+
+ gitInfoExtension
+ .getDotGitDir()
.convention(
- providers.provider(
- () -> {
- Directory projectDirectory =
- project.getRootProject().getLayout().getProjectDirectory();
- Path gitLocation = projectDirectory.getAsFile().toPath().resolve(".git");
- if (!Files.exists(gitLocation)) {
- // don't return anything from the provider if we can't locate the .git
- // folder. This will result in the property returning false from isPresent.
- return null;
- }
+ project
+ .getProviders()
+ .provider(
+ () -> {
+ Directory projectDirectory =
+ project.getRootProject().getLayout().getProjectDirectory();
+ Path gitLocation = projectDirectory.getAsFile().toPath().resolve(".git");
+ if (!Files.exists(gitLocation)) {
+ // don't return anything from the provider if we can't locate the .git
+ // folder. This will result in the property returning false from isPresent.
+ return null;
+ }
- if (Files.isDirectory(gitLocation)) {
- return projectDirectory.dir(".git");
- } else if (Files.isRegularFile(gitLocation)) {
- return projectDirectory.file(".git");
- } else {
- throw new GradleException(
- "Panic, .git location not a directory or file: "
- + gitLocation.toAbsolutePath());
- }
- }))
+ if (Files.isDirectory(gitLocation)) {
+ return projectDirectory.dir(".git");
+ } else if (Files.isRegularFile(gitLocation)) {
+ return projectDirectory.file(".git");
+ } else {
+ throw new GradleException(
+ "Panic, .git location not a directory or file: "
+ + gitLocation.toAbsolutePath());
+ }
+ }))
.finalizeValueOnRead();
-
- var gitExec =
- project.getExtensions().getByType(LuceneBuildGlobalsExtension.class).externalTool("git");
-
- Action<ValueSourceSpec<GitValueSourceParameters>> configureGitParams =
- spec -> {
- var params = spec.getParameters();
- params.getRootProjectDir().set(project.getProjectDir());
- params.getGitExec().set(gitExec);
- params.getDotDir().set(dotGitDir);
- };
-
- gitInfoExtension
- .getGitInfo()
- .value(providers.of(GitInfoValueSource.class, configureGitParams))
- .finalizeValueOnRead();
-
- gitInfoExtension
- .getAllNonIgnoredProjectFiles()
- .value(providers.of(GitFileListValueSource.class, configureGitParams));
}
}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java
index 0f1cae5..cd10457 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java
@@ -25,12 +25,19 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.inject.Inject;
+import org.apache.tools.ant.taskdefs.condition.Os;
import org.gradle.api.GradleException;
+import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.provider.ValueSource;
+import org.gradle.api.provider.ValueSourceParameters;
+import org.gradle.api.services.BuildServiceParameters;
import org.gradle.process.ExecOperations;
public abstract class GitInfoValueSource
- implements ValueSource<Map<String, String>, GitValueSourceParameters> {
+ implements ValueSource<Map<String, String>, GitInfoValueSource.Parameters> {
+ public abstract static class Parameters implements BuildServiceParameters, ValueSourceParameters {
+ public abstract DirectoryProperty getRootProjectDir();
+ }
@Inject
public abstract ExecOperations getExecOps();
@@ -47,9 +54,9 @@
spec.setErrorOutput(out);
spec.setIgnoreExitValue(true);
- getParameters().configure(spec);
-
- spec.setExecutable(getParameters().getGitExec().get());
+ spec.setWorkingDir(
+ getParameters().getRootProjectDir().getAsFile().get().getAbsolutePath());
+ spec.setExecutable(Os.isFamily(Os.FAMILY_WINDOWS) ? "git.exe" : "git");
spec.args("status", "--porcelain=v2", "--branch");
});
out.flush();
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java
deleted file mode 100644
index 49b5726..0000000
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.gradle.plugins.gitinfo;
-
-import org.gradle.api.file.DirectoryProperty;
-import org.gradle.api.file.FileSystemLocation;
-import org.gradle.api.provider.Property;
-import org.gradle.api.provider.ValueSourceParameters;
-import org.gradle.api.services.BuildServiceParameters;
-import org.gradle.process.ExecSpec;
-
-public abstract class GitValueSourceParameters
- implements BuildServiceParameters, ValueSourceParameters {
- public abstract DirectoryProperty getRootProjectDir();
-
- public abstract Property<String> getGitExec();
-
- public abstract Property<FileSystemLocation> getDotDir();
-
- void configure(ExecSpec spec) {
- String workDir = getRootProjectDir().getAsFile().get().getAbsolutePath();
- spec.setWorkingDir(workDir);
- spec.environment("GIT_DIR", getDotDir().get().getAsFile().getAbsolutePath());
- }
-}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java
index 224e343..628c490 100644
--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java
@@ -16,13 +16,8 @@
*/
package org.apache.lucene.gradle.plugins.licenses;
-import java.io.File;
-import java.util.List;
import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;
-import org.apache.lucene.gradle.plugins.gitinfo.GitInfoExtension;
-import org.apache.tools.ant.types.selectors.SelectorUtils;
import org.gradle.api.Project;
-import org.gradle.api.specs.Spec;
/** This configures ASL and other license checks. */
public class CheckLicensesPlugin extends LuceneGradlePlugin {
@@ -53,96 +48,79 @@
private void configureCheckLicenses(CheckLicensesTask task) {
Project project = task.getProject();
- assert project.getRootProject() == project;
- var allNonIgnoredFiles =
- project.getExtensions().getByType(GitInfoExtension.class).getAllNonIgnoredProjectFiles();
-
task.getReportFile().set(project.getLayout().getBuildDirectory().file("licenses-report.txt"));
+ task.getFiles()
+ .from(
+ project.fileTree(
+ ".",
+ tree -> {
+ // Exclude build outputs, ide files, .git.
+ tree.exclude(".git");
+ tree.exclude(".idea");
+ tree.exclude(".muse");
+ tree.exclude("**/build/**");
+ tree.exclude("**/.gradle");
- // Build a list of files excluded from the license check. Just reuse ant's glob patterns.
- var rootDir = getProjectRootPath(project);
- var excludedPaths =
- List.of(
- // Ignore binary files. Previously we used apache rat, which had a 'binary guesser'
- // but it's faster to just exclude by name (rather than scan the file).
- "**/*.adoc",
- "**/*.bin",
- "**/*.brk",
- "**/*.bz2",
- "**/*.dat",
- "**/*.gif",
- "**/*.gz",
- "**/*.png",
- "**/*.svg",
- "**/*.xls",
- "**/*.zip",
+ // Exclude generated stuff.
+ tree.exclude("**/src/generated/**");
- // Ignore build infrastructure and misc utility files.
- ".asf.yaml",
- ".dir-locals.el",
- ".editorconfig",
- ".git-blame-ignore-revs",
- ".gitattributes",
- ".github/**",
- ".gitignore",
- ".lift.toml",
- ".vscode/**",
- "LICENSE.txt",
- "NOTICE.txt",
- "build-options.properties",
- "dev-tools/**",
- "gradle/**",
- "help/*.txt",
- "lucene/licenses/*",
- "versions.lock",
+ // Exclude github stuff (templates, workflows).
+ tree.exclude(".github");
- // Ignore resources in source folders, also generated resources.
- "**/src/**/*.txt",
- "**/src/**/*.properties",
- "**/src/**/*.utf8",
- "**/src/generated/**",
+ // do not let RAT attempt to scan a python venv, it gets lost and confused...
+ tree.exclude("**/.venv/**");
- // Ignore other binary resources within sources. Try to be
- // specific here.
- "build-tools/build-infra-shadow/src/java/keep.me",
- "lucene/CHANGES.txt",
- "lucene/analysis.tests/src/**/*.aff",
- "lucene/analysis.tests/src/**/*.dic",
- "lucene/analysis/common/src/**/*.aff",
- "lucene/analysis/common/src/**/*.dic",
- "lucene/analysis/common/src/**/*.good",
- "lucene/analysis/common/src/**/*.htm*",
- "lucene/analysis/common/src/**/*.rslp",
- "lucene/analysis/common/src/**/*.sug",
- "lucene/analysis/common/src/**/*.wrong",
- "lucene/analysis/icu/src/**/utr30.nrm",
- "lucene/analysis/kuromoji/src/**/bocchan.utf-8",
- "lucene/analysis/morfologik/src/**/*.dict",
- "lucene/analysis/morfologik/src/**/*.info",
- "lucene/analysis/morfologik/src/**/*.input",
- "lucene/analysis/opennlp/src/**/en-test-lemmas.dict",
- "lucene/analysis/smartcn/src/**/*.mem",
- "lucene/analysis/stempel/src/**/*.tbl",
- "lucene/benchmark/.gitignore",
- "lucene/demo/src/**/knn-token-vectors",
- "lucene/luke/src/**/ElegantIcons.ttf",
- "lucene/test-framework/src/**/europarl.lines.txt.seek",
+ // apache rat has a 'binary guesser'... I don't think this needs to be done at all
+ // -
+ // just exclude binaries here.
+ tree.exclude("**/*.dat");
+ tree.exclude("**/*.brk");
+ tree.exclude("**/*.gz");
+ tree.exclude("**/*.bin");
+ tree.exclude("**/*.bz2");
+ tree.exclude("**/*.gif");
+ tree.exclude("**/*.svg");
+ tree.exclude("lucene/analysis/smartcn/src/**/*.mem");
- // these may require a review, actually?
- "lucene/queryparser/docs/**",
- "lucene/benchmark/conf/*",
- "lucene/benchmark/README.enwiki");
+ // Only check these selected file patterns as folks have various .gitignore-d
+ // resources generated by IDEs, etc.
+ tree.include("**/*.gradle");
+ tree.include("**/*.xml");
+ tree.include("**/*.md");
+ tree.include("**/*.py");
+ tree.include("**/*.sh");
+ tree.include("**/*.bat");
- // I thought it'd be possible to somehow precompile those glob filters but apparently not.
- // I guess it's fine if the list of patterns and files is of reasonable size.
- Spec<File> maybeExcludeKnownExceptions =
- file -> {
- // relativize from root project directory. No need to replace path separators as ant is
- // os-agnostic here.
- var filePath = rootDir.relativize(file.toPath()).toString();
- return excludedPaths.stream()
- .noneMatch(pattern -> SelectorUtils.match(pattern, filePath));
- };
- task.getFiles().from(project.files(allNonIgnoredFiles).filter(maybeExcludeKnownExceptions));
+ // Include selected patterns from any source folders.
+ tree.include("**/src/**");
+ tree.exclude("**/src/**/*.png");
+ tree.exclude("**/src/**/*.txt");
+ tree.exclude("**/src/**/*.zip");
+ tree.exclude("**/src/**/*.properties");
+ tree.exclude("**/src/**/*.utf8");
+
+ // project-specific exclusions.
+ tree.exclude("build-tools/build-infra-shadow/src/java/keep.me");
+ tree.exclude("lucene/analysis/icu/src/**/utr30.nrm");
+ tree.exclude("lucene/analysis/kuromoji/src/**/bocchan.utf-8");
+ tree.exclude("lucene/analysis/morfologik/src/**/*.info");
+ tree.exclude("lucene/analysis/morfologik/src/**/*.input");
+ tree.exclude("lucene/analysis/morfologik/src/**/*.dict");
+ tree.exclude("lucene/analysis/stempel/src/**/*.tbl");
+ tree.exclude("lucene/analysis/opennlp/src/**/en-test-lemmas.dict");
+ tree.exclude("lucene/demo/src/**/knn-token-vectors");
+ tree.exclude("lucene/test-framework/src/**/europarl.lines.txt.seek");
+ tree.exclude("lucene/analysis/common/src/**/*.aff");
+ tree.exclude("lucene/analysis/common/src/**/*.dic");
+ tree.exclude("lucene/analysis/common/src/**/*.good");
+ tree.exclude("lucene/analysis/common/src/**/*.sug");
+ tree.exclude("lucene/analysis/common/src/**/*.wrong");
+ tree.exclude("lucene/analysis/common/src/**/*.rslp");
+ tree.exclude("lucene/analysis/common/src/**/*.htm*");
+ tree.exclude("lucene/analysis.tests/src/**/*.aff");
+ tree.exclude("lucene/analysis.tests/src/**/*.dic");
+ // Luke has an embedded ElegantIcons font (MIT licensed).
+ tree.exclude("lucene/luke/src/**/ElegantIcons.ttf");
+ }));
}
}
diff --git a/build.gradle b/build.gradle
index b483f83..926c584 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,4 +1,3 @@
-import org.apache.lucene.gradle.plugins.gitinfo.GitInfoExtension
import org.apache.lucene.gradle.plugins.licenses.CheckLicensesTask
/*
