diff --git a/.bazelrc b/.bazelrc index b309d74c80091d..3225f305600245 100644 --- a/.bazelrc +++ b/.bazelrc @@ -52,10 +52,13 @@ build:windows --per_file_copt=external/.*@/w build:windows --host_per_file_copt=external/.*@/w # Enable Java 21 language features -build --java_runtime_version=21 -build --java_language_version=21 -build --tool_java_language_version=21 -build --tool_java_runtime_version=21 +common --java_runtime_version=21 +common --java_language_version=21 +common --tool_java_language_version=21 +common --tool_java_runtime_version=21 +# Ensure that Java language level and bootclasspath version are the same. +# This avoids accidental usage of more recent Java APIs. +common --@rules_java//toolchains:incompatible_language_version_bootclasspath # User-specific .bazelrc try-import %workspace%/user.bazelrc diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 4e282951a642e5..61e619c27027db 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -236,7 +236,7 @@ "moduleExtensions": { "//:repositories.bzl%async_profiler_repos": { "general": { - "bzlTransitiveDigest": "Yke2x5sUjmlG+2GhGPMhz8ErEEO8cBJUYJ/IlxyQcbA=", + "bzlTransitiveDigest": "kVJNKdqc8D0Cno3DHf1yNSuKrItg5PBceuXeuS6q+HI=", "usagesDigest": "fv/Ru+up/1CLlox7G1yNn9y4JOVK1qeU6uQCAkCcMaM=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -422,6 +422,11 @@ "rules_shell", "rules_shell+" ], + [ + "", + "with_cfg.bzl", + "with_cfg.bzl+" + ], [ "", "zlib", diff --git a/repositories.bzl b/repositories.bzl index 0b09c9a7cc5996..e9ee5205b936c7 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -51,6 +51,7 @@ DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [ "rules_proto", "rules_python", "rules_shell", + "with_cfg.bzl", "zlib", "zstd-jni", ]] + [(get_canonical_repo_name("com_github_grpc_grpc") + "+grpc_repo_deps_ext+" + suffix) for suffix in [ diff --git a/src/BUILD b/src/BUILD index 11426f78cec191..d917fba34b9993 100644 --- a/src/BUILD +++ b/src/BUILD @@ -6,8 +6,7 @@ load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_java//java:java_binary.bzl", "java_binary") load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load("//src:build_defs.bzl", "transition_java_language_8_archive") -load("//src:release_archive.bzl", "release_archive") +load("//src:release_archive.bzl", "minimum_java_compilation_runtime_filegroup", "minimum_java_runtime_filegroup", "release_archive") load(":embedded_tools.bzl", "srcsfile") load(":rule_size_test.bzl", "rule_size_test") @@ -512,19 +511,33 @@ genrule( ) # Following targets build java_tools.zip - platform independent part of java_tools -JAVA_TOOLS_DEPLOY_JARS = [ - "//src/java_tools/buildjar:JavaBuilder_deploy.jar", - "//src/java_tools/buildjar:VanillaJavaBuilder_deploy.jar", - "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar", - "//src/java_tools/buildjar/java/com/google/devtools/build/java/turbine:turbine_direct_binary_deploy.jar", - "//src/java_tools/import_deps_checker/java/com/google/devtools/build/importdeps:ImportDepsChecker_deploy.jar", - "//src/java_tools/junitrunner/java/com/google/testing/coverage:JacocoCoverage_jarjar_deploy.jar", - "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar", -] +minimum_java_runtime_filegroup( + name = "minimum_java_runtime_tools", + srcs = [ + "//src/java_tools/junitrunner/java/com/google/testing/coverage:JacocoCoverage_jarjar_deploy.jar", + "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar", + ], + visibility = ["//visibility:private"], +) + +minimum_java_compilation_runtime_filegroup( + name = "minimum_java_compilation_runtime_tools", + srcs = [ + "//src/java_tools/buildjar:JavaBuilder_deploy.jar", + "//src/java_tools/buildjar:VanillaJavaBuilder_deploy.jar", + "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar", + "//src/java_tools/buildjar/java/com/google/devtools/build/java/turbine:turbine_direct_binary_deploy.jar", + "//src/java_tools/import_deps_checker/java/com/google/devtools/build/importdeps:ImportDepsChecker_deploy.jar", + ], + visibility = ["//visibility:private"], +) release_archive( name = "jars_java_tools_zip", - srcs = JAVA_TOOLS_DEPLOY_JARS, + srcs = [ + ":minimum_java_compilation_runtime_tools", + ":minimum_java_runtime_tools", + ], package_dir = "java_tools", visibility = ["//visibility:private"], ) @@ -545,12 +558,6 @@ release_archive( ], ) -transition_java_language_8_archive( - name = "java_tools_java8.zip", - archive_zip = ":java_tools.zip", - visibility = ["//src/test/shell/bazel:__pkg__"], -) - release_archive( name = "turbine_direct_graal_zip", srcs = ["//src/java_tools/buildjar/java/com/google/devtools/build/java/turbine:turbine_direct_graal"], @@ -573,12 +580,6 @@ release_archive( ], ) -transition_java_language_8_archive( - name = "java_tools_prebuilt_java8.zip", - archive_zip = ":java_tools_prebuilt.zip", - visibility = ["//src/test/shell/bazel:__pkg__"], -) - # Following targets used by the java_tools_binaries Buildkite pipeline to upload # the java_tools_*.zip to either tmp/sources or tmp/build directories in GCS. sh_binary( @@ -664,11 +665,6 @@ filegroup( ], ) -bzl_library( - name = "build_defs_bzl", - srcs = ["build_defs.bzl"], -) - java_binary( name = "CheckSunJnuEncoding", srcs = ["CheckSunJnuEncoding.java"], diff --git a/src/build_defs.bzl b/src/build_defs.bzl deleted file mode 100644 index 917f434a7941db..00000000000000 --- a/src/build_defs.bzl +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility for restricting Java APIs.""" - -_java_language_version_8_transition = transition( - implementation = lambda settings, attr: { - "//command_line_option:java_language_version": "8", - }, - inputs = [], - outputs = ["//command_line_option:java_language_version"], -) - -def _transition_java_language_8_archive_impl(ctx): - archive_zip = ctx.files.archive_zip[0] - - outfile = ctx.actions.declare_file(ctx.label.name) - - ctx.actions.run_shell( - inputs = [archive_zip], - outputs = [outfile], - command = "cp %s %s" % (archive_zip.path, outfile.path), - ) - return [ - DefaultInfo( - files = depset([outfile]), - ), - ] - -_transitioned_java_8_archive = rule( - implementation = _transition_java_language_8_archive_impl, - attrs = { - "archive_zip": attr.label( - allow_files = True, - cfg = _java_language_version_8_transition, - mandatory = True, - ), - }, -) - -# Used to transition the zip file generated by release_archive to compile at Java language 8. -def transition_java_language_8_archive(name, archive_zip, visibility): - _transitioned_java_8_archive( - name = name, - archive_zip = archive_zip, - visibility = visibility, - ) diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java index 161bb0344326d7..20bd1ea7df4168 100644 --- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java @@ -297,8 +297,7 @@ private static void setLocations(JavacFileManager fileManager, BlazeJavacArgumen try { fileManager.setLocationFromPaths(StandardLocation.CLASS_PATH, arguments.classPath()); // modular dependencies must be on the module path, not the classpath - fileManager.setLocationFromPaths( - StandardLocation.locationFor("MODULE_PATH"), arguments.classPath()); + fileManager.setLocationFromPaths(StandardLocation.MODULE_PATH, arguments.classPath()); fileManager.setLocationFromPaths( StandardLocation.CLASS_OUTPUT, ImmutableList.of(arguments.classOutput())); @@ -326,7 +325,7 @@ private static void setLocations(JavacFileManager fileManager, BlazeJavacArgumen Path system = arguments.system(); if (system != null) { fileManager.setLocationFromPaths( - StandardLocation.locationFor("SYSTEM_MODULES"), ImmutableList.of(system)); + StandardLocation.SYSTEM_MODULES, ImmutableList.of(system)); } // The bootclasspath may legitimately be empty if --release is being used. Collection bootClassPath = arguments.bootClassPath(); diff --git a/src/java_tools/junitrunner/java/com/google/testing/junit/runner/internal/SystemExitDetectingShutdownHook.java b/src/java_tools/junitrunner/java/com/google/testing/junit/runner/internal/SystemExitDetectingShutdownHook.java index cc48db404bb6c3..b0bec56e0964a2 100644 --- a/src/java_tools/junitrunner/java/com/google/testing/junit/runner/internal/SystemExitDetectingShutdownHook.java +++ b/src/java_tools/junitrunner/java/com/google/testing/junit/runner/internal/SystemExitDetectingShutdownHook.java @@ -14,10 +14,8 @@ package com.google.testing.junit.runner.internal; -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toList; - import java.io.PrintStream; +import java.util.ArrayList; import java.util.List; /** @@ -28,40 +26,41 @@ */ public class SystemExitDetectingShutdownHook { public static Thread newShutdownHook(PrintStream testRunnerOut) { - Runnable hook = () -> { - boolean foundRuntimeExit = false; - for (StackTraceElement[] stack : Thread.getAllStackTraces().values()) { - @SuppressWarnings("JdkCollectors") // can't use ImmutableList here - List framesStartingWithRuntimeExit = - stream(stack) - .dropWhile( - frame -> - !frame.getClassName().equals("java.lang.Runtime") - || !frame.getMethodName().equals("exit")) - .map(SystemExitDetectingShutdownHook::frameString) - .collect(toList()); - if (!framesStartingWithRuntimeExit.isEmpty()) { - foundRuntimeExit = true; - testRunnerOut.println("\nSystem.exit or Runtime.exit was called!"); - testRunnerOut.println(String.join("\n", framesStartingWithRuntimeExit)); - } - } - if (foundRuntimeExit) { - // We must call halt rather than exit, because exit would lead to a deadlock. We use a - // hopefully unique exit code to make it easier to identify this case. - Runtime.getRuntime().halt(121); - } - }; + Runnable hook = + () -> { + boolean foundRuntimeExit = false; + for (StackTraceElement[] stack : Thread.getAllStackTraces().values()) { + List framesStartingWithRuntimeExit = new ArrayList<>(); + boolean foundRuntimeExitInThisThread = false; + for (StackTraceElement frame : stack) { + if (!foundRuntimeExitInThisThread + && frame.getClassName().equals("java.lang.Runtime") + && frame.getMethodName().equals("exit")) { + foundRuntimeExitInThisThread = true; + } + if (foundRuntimeExitInThisThread) { + framesStartingWithRuntimeExit.add(frameString(frame)); + } + } + if (foundRuntimeExitInThisThread) { + foundRuntimeExit = true; + testRunnerOut.println("\nSystem.exit or Runtime.exit was called!"); + testRunnerOut.println(String.join("\n", framesStartingWithRuntimeExit)); + } + } + if (foundRuntimeExit) { + // We must call halt rather than exit, because exit would lead to a deadlock. We use a + // hopefully unique exit code to make it easier to identify this case. + Runtime.getRuntime().halt(121); + } + }; return new Thread(hook, "SystemExitDetectingShutdownHook"); } private static String frameString(StackTraceElement frame) { return String.format( " at %s.%s(%s:%d)", - frame.getClassName(), - frame.getMethodName(), - frame.getFileName(), - frame.getLineNumber()); + frame.getClassName(), frame.getMethodName(), frame.getFileName(), frame.getLineNumber()); } private SystemExitDetectingShutdownHook() {} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/config/BUILD index ca7296cd6b387f..9f6065063f6b7a 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/BUILD +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BUILD @@ -246,6 +246,7 @@ java_library( name = "execution_info_modifier", srcs = ["ExecutionInfoModifier.java"], deps = [ + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/common/options", "//third_party:auto_value", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ExecutionInfoModifier.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ExecutionInfoModifier.java index 33894fb577614f..512022907229c8 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/ExecutionInfoModifier.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ExecutionInfoModifier.java @@ -14,6 +14,8 @@ package com.google.devtools.build.lib.analysis.config; +import static com.google.devtools.build.lib.util.StringEncoding.internalToUnicode; + import com.google.auto.value.AutoValue; import com.google.auto.value.extension.memoized.Memoized; import com.google.common.base.Splitter; @@ -102,7 +104,8 @@ private static ExecutionInfoModifier create(String input, ImmutableList expr.pattern().matcher(mnemonic).matches()); + return expressions().stream() + .anyMatch(expr -> expr.pattern().matcher(internalToUnicode(mnemonic)).matches()); } /** Checks whether the {@code executionInfoList} matches the {@code mnemonic}. */ @@ -139,7 +142,7 @@ public static void apply( /** Modifies the given map of {@code executionInfo} to add or remove the keys for this option. */ void apply(String mnemonic, Map executionInfo) { for (Expression expr : expressions()) { - if (expr.pattern().matcher(mnemonic).matches()) { + if (expr.pattern().matcher(internalToUnicode(mnemonic)).matches()) { if (expr.remove()) { executionInfo.remove(expr.key()); } else { diff --git a/src/main/java/com/google/devtools/build/lib/events/BUILD b/src/main/java/com/google/devtools/build/lib/events/BUILD index 8b805673b9fdf3..1c1679654668f3 100644 --- a/src/main/java/com/google/devtools/build/lib/events/BUILD +++ b/src/main/java/com/google/devtools/build/lib/events/BUILD @@ -15,6 +15,7 @@ java_library( name = "events", srcs = glob(["*.java"]), deps = [ + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/build/lib/util/io:out-err", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", diff --git a/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java b/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java index bea4ddf190ed26..3fd2b2042934de 100644 --- a/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java +++ b/src/main/java/com/google/devtools/build/lib/events/OutputFilter.java @@ -14,32 +14,19 @@ package com.google.devtools.build.lib.events; +import com.google.devtools.build.lib.util.StringEncoding; import java.util.regex.Pattern; -/** - * An output filter for warnings. - */ +/** An output filter for warnings. */ public interface OutputFilter { /** An output filter that matches everything. */ - public static final OutputFilter OUTPUT_EVERYTHING = new OutputFilter() { - @Override - public boolean showOutput(String tag) { - return true; - } - }; + OutputFilter OUTPUT_EVERYTHING = tag -> true; /** An output filter that matches nothing. */ - public static final OutputFilter OUTPUT_NOTHING = new OutputFilter() { - @Override - public boolean showOutput(String tag) { - return false; - } - }; + OutputFilter OUTPUT_NOTHING = tag -> false; - /** - * Returns true iff the given tag matches the output filter. - */ + /** Returns true iff the given tag matches the output filter. */ boolean showOutput(String tag); /** An output filter using regular expression matching. */ @@ -62,7 +49,7 @@ private RegexOutputFilter(Pattern pattern) { @Override public boolean showOutput(String tag) { - return pattern.matcher(tag).find(); + return pattern.matcher(StringEncoding.internalToUnicode(tag)).find(); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/BUILD b/src/main/java/com/google/devtools/build/lib/exec/local/BUILD index 6a0756132f9c5c..fdcc87062a01c8 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/BUILD +++ b/src/main/java/com/google/devtools/build/lib/exec/local/BUILD @@ -39,6 +39,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/shell", "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/build/lib/util/io", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/protobuf:failure_details_java_proto", diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java index 4e0d5e1e058df9..68224b27d6f3ed 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java @@ -55,6 +55,7 @@ import com.google.devtools.build.lib.shell.SubprocessBuilder; import com.google.devtools.build.lib.shell.TerminationStatus; import com.google.devtools.build.lib.util.NetUtil; +import com.google.devtools.build.lib.util.StringEncoding; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; import com.google.errorprone.annotations.FormatMethod; @@ -265,7 +266,7 @@ private SpawnResult runOnce() throws InterruptedException, ExecException, IOExce @FormatMethod private void stepLog(Level level, @FormatString String fmt, Object... args) { - stepLog(level, /*cause=*/ null, fmt, args); + stepLog(level, /* cause= */ null, fmt, args); } @FormatMethod @@ -322,7 +323,8 @@ private SpawnResult start() throws InterruptedException, ExecException, IOExcept ("Action type " + actionType + " is not allowed to run locally due to regex filter: " - + localExecutionOptions.allowedLocalAction.regexPattern() + + StringEncoding.unicodeToInternal( + localExecutionOptions.allowedLocalAction.regexPattern().toString()) + "\n") .getBytes(UTF_8)); spawnMetrics.setTotalTime(totalTimeStopwatch.elapsed()); diff --git a/src/main/java/com/google/devtools/build/lib/unsafe/StringUnsafe.java b/src/main/java/com/google/devtools/build/lib/unsafe/StringUnsafe.java index a4044e30142f76..1e42d7706dbc29 100644 --- a/src/main/java/com/google/devtools/build/lib/unsafe/StringUnsafe.java +++ b/src/main/java/com/google/devtools/build/lib/unsafe/StringUnsafe.java @@ -91,8 +91,9 @@ public static byte[] getInternalStringBytes(String obj) { // Truncation is ASCII only and thus doesn't change the encoding. String truncatedString = Ascii.truncate(obj, 1000, "..."); throw new IllegalArgumentException( - "Expected internal string with Latin-1 coder, got: %s (%s)" - .formatted(truncatedString, Arrays.toString(getByteArray(truncatedString)))); + String.format( + "Expected internal string with Latin-1 coder, got: %s (%s)", + truncatedString, Arrays.toString(getByteArray(truncatedString)))); } return getByteArray(obj); } diff --git a/src/main/java/com/google/devtools/build/lib/util/regex/BUILD b/src/main/java/com/google/devtools/build/lib/util/regex/BUILD index f6bbaca854a6db..6f58114741e772 100644 --- a/src/main/java/com/google/devtools/build/lib/util/regex/BUILD +++ b/src/main/java/com/google/devtools/build/lib/util/regex/BUILD @@ -14,4 +14,7 @@ filegroup( java_library( name = "regex_util", srcs = ["RegexUtil.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", + ], ) diff --git a/src/main/java/com/google/devtools/build/lib/util/regex/RegexUtil.java b/src/main/java/com/google/devtools/build/lib/util/regex/RegexUtil.java index 349e021f0561cd..bcaa77a7143d9e 100644 --- a/src/main/java/com/google/devtools/build/lib/util/regex/RegexUtil.java +++ b/src/main/java/com/google/devtools/build/lib/util/regex/RegexUtil.java @@ -13,6 +13,9 @@ // limitations under the License. package com.google.devtools.build.lib.util.regex; +import static com.google.devtools.build.lib.util.StringEncoding.internalToUnicode; +import static com.google.devtools.build.lib.util.StringEncoding.unicodeToInternal; + import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -31,7 +34,8 @@ public final class RegexUtil { Pattern.compile("(?:\\^|\\\\A)*\\.\\*(?[^?+].*)?"); /** - * Returns a {@link Predicate} that matches the input string against the regex pattern with the + * Returns a {@link Predicate} that matches the input string (in internal encoding according to + * {@link com.google.devtools.build.lib.util.StringEncoding}) against the regex pattern with the * same semantics as {@link Matcher#matches()}, but more optimized and as if the pattern was * compiled with {@link Pattern#DOTALL}. */ @@ -43,7 +47,11 @@ public static Predicate asOptimizedMatchingPredicate(Pattern regexPatter // engine's optimization pass exploited below - it only applies when the first node is a // literal, not a group. // Unmatched \Q...\E sequences can have the same effect. - return s -> regexPattern.matcher(s).matches(); + return s -> regexPattern.matcher(internalToUnicode(s)).matches(); + } + // If the pattern matches a literal string, optimize to an equals call. + if (LITERAL_PATTERN_WITH_DOT_UNESCAPED.matcher(pattern).matches()) { + return unicodeToInternal(pattern.replace("\\.", "."))::equals; } // Recognize a pattern that starts with ".*" and drop it so that the engine sees the next part // of the pattern as the beginning and potentially optimizes the literal search. @@ -51,7 +59,7 @@ public static Predicate asOptimizedMatchingPredicate(Pattern regexPatter // determine if a ".*" suffix is safe to drop due to backslash escapes. Matcher suffixMatch = EXTRACT_SUFFIX_MATCH.matcher(pattern); if (!suffixMatch.matches()) { - return s -> regexPattern.matcher(s).matches(); + return s -> regexPattern.matcher(internalToUnicode(s)).matches(); } String suffixPattern = suffixMatch.group("suffix"); // A null suffixPattern implies the regex is equivalent to ".*", which matches any input string. @@ -61,14 +69,14 @@ public static Predicate asOptimizedMatchingPredicate(Pattern regexPatter // If the pattern matches a literal suffix, optimize to a string suffix // match, which is by far the fastest way to match. if (LITERAL_PATTERN_WITH_DOT_UNESCAPED.matcher(suffixPattern).matches()) { - String literalSuffix = suffixPattern.replace("\\.", "."); + String literalSuffix = unicodeToInternal(suffixPattern.replace("\\.", ".")); return s -> s.endsWith(literalSuffix); } // Turn the "match" pattern into an equivalent "find" pattern, since these are the only ones // that benefit from the Boyer-Moore optimization in the Java regex engine. // https://github.com/openjdk/jdk/blob/50dced88ff1aed23bb4c8fe9e4a08e6cc200b897/src/java.base/share/classes/java/util/regex/Pattern.java#L1959-L1969 Pattern compiled = Pattern.compile(suffixPattern + "\\z", Pattern.DOTALL); - return s -> compiled.matcher(s).find(); + return s -> compiled.matcher(internalToUnicode(s)).find(); } private RegexUtil() {} diff --git a/src/main/java/com/google/devtools/common/options/BUILD b/src/main/java/com/google/devtools/common/options/BUILD index 72975c2eab1056..3e8acf9d01f2af 100644 --- a/src/main/java/com/google/devtools/common/options/BUILD +++ b/src/main/java/com/google/devtools/common/options/BUILD @@ -50,6 +50,7 @@ java_library( ), deps = [ "//src/main/java/com/google/devtools/build/lib/util:pair", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/build/lib/util/regex:regex_util", "//src/main/java/net/starlark/java/spelling", "//third_party:auto_value", diff --git a/src/main/java/com/google/devtools/common/options/Converters.java b/src/main/java/com/google/devtools/common/options/Converters.java index d6092b9e3bea24..4bfc8c5279b727 100644 --- a/src/main/java/com/google/devtools/common/options/Converters.java +++ b/src/main/java/com/google/devtools/common/options/Converters.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; +import com.google.devtools.build.lib.util.StringEncoding; import java.time.Duration; import java.util.Iterator; import java.util.List; @@ -422,7 +423,8 @@ public static class RegexPatternConverter extends Converter.ContextlessNote: Strings passed to the {@link Pattern} and {@link java.util.regex.Matcher} API have to + * be converted to "Unicode" form first (see {@link + * com.google.devtools.build.lib.util.StringEncoding#internalToUnicode}. + */ public abstract Pattern regexPattern(); /** diff --git a/src/release_archive.bzl b/src/release_archive.bzl index 32395abcf0a9ac..eda6c1edd85312 100644 --- a/src/release_archive.bzl +++ b/src/release_archive.bzl @@ -14,6 +14,37 @@ """Rules to create a release archive""" +load("@rules_java//java:java_binary.bzl", "java_binary") +load("@with_cfg.bzl", "with_cfg") + +# The minimum --java_{tool_,}runtime_version supported by prebuilt Java tools. +_MINIMUM_JAVA_RUNTIME_VERSION = 8 + +# The minimum version of a java_toolchain's java_runtime supported by prebuilt Java tools. +_MINIMUM_JAVA_COMPILATION_RUNTIME_VERSION = 11 + +minimum_java_runtime_java_binary, _minimum_java_runtime_java_binary = ( + # Don't warn about targeting very old Java versions. + with_cfg(java_binary) + .set("java_language_version", str(_MINIMUM_JAVA_RUNTIME_VERSION)) + .extend("javacopt", ["-Xlint:-options"]) + .build() +) + +minimum_java_runtime_filegroup, _minimum_java_runtime_filegroup = ( + # Don't warn about targeting very old Java versions. + with_cfg(native.filegroup) + .set("java_language_version", str(_MINIMUM_JAVA_RUNTIME_VERSION)) + .extend("javacopt", ["-Xlint:-options"]) + .build() +) + +minimum_java_compilation_runtime_filegroup, _minimum_java_compilation_runtime_filegroup = ( + with_cfg(native.filegroup) + .set("java_language_version", str(_MINIMUM_JAVA_COMPILATION_RUNTIME_VERSION)) + .build() +) + def release_archive(name, srcs = [], src_map = {}, package_dir = "-", deps = [], **kwargs): """ Creates an zip of the srcs, and renamed label artifacts. diff --git a/src/test/java/com/google/devtools/build/lib/util/regex/BUILD b/src/test/java/com/google/devtools/build/lib/util/regex/BUILD index 5f1af94552fa86..3d1ca90295f1a2 100644 --- a/src/test/java/com/google/devtools/build/lib/util/regex/BUILD +++ b/src/test/java/com/google/devtools/build/lib/util/regex/BUILD @@ -38,7 +38,7 @@ java_binary( ) # Start fuzzing via -# bazel run //src/test/java/com/google/devtools/build/lib/util:RegexUtilFuzzTest_run +# bazel run //src/test/java/com/google/devtools/build/lib/util/regex:RegexUtilFuzzTest_run java_fuzz_test( name = "RegexUtilFuzzTest", srcs = ["RegexUtilFuzzTest.java"], @@ -53,6 +53,7 @@ java_fuzz_test( "notap", ], deps = [ + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/build/lib/util/regex:regex_util", "//third_party:jmh", ], diff --git a/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilFuzzTest.java b/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilFuzzTest.java index ecac6ba0f3beb4..cac2c4c521c624 100644 --- a/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilFuzzTest.java +++ b/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilFuzzTest.java @@ -13,6 +13,8 @@ // limitations under the License. package com.google.devtools.build.lib.util.regex; +import static com.google.devtools.build.lib.util.StringEncoding.unicodeToInternal; + import com.code_intelligence.jazzer.api.FuzzedDataProvider; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -35,7 +37,9 @@ public static void fuzzerTestOneInput(FuzzedDataProvider data) { } Predicate optimizedMatcher = RegexUtil.asOptimizedMatchingPredicate(originalPattern); - if (optimizedMatcher.test(haystack) != originalPattern.matcher(haystack).matches()) { + boolean originalMatches = originalPattern.matcher(haystack).matches(); + boolean optimizedMatches = optimizedMatcher.test(unicodeToInternal(haystack)); + if (originalMatches != optimizedMatches) { throw new AssertionError( """ Optimized matcher and original matcher differ in behavior: @@ -44,11 +48,7 @@ public static void fuzzerTestOneInput(FuzzedDataProvider data) { originalPattern.matcher(haystack).matches(): %s optimizedMatcher.test(haystack): %s """ - .formatted( - needle, - haystack, - originalPattern.matcher(haystack).matches(), - optimizedMatcher.test(haystack))); + .formatted(needle, haystack, originalMatches, optimizedMatches)); } } } diff --git a/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilTest.java b/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilTest.java index ae502dc0fabf1a..f6114b80c5c2e3 100644 --- a/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilTest.java +++ b/src/test/java/com/google/devtools/build/lib/util/regex/RegexUtilTest.java @@ -34,6 +34,7 @@ public void optimizedMatchingPredicate( "a", "foo", "foofoo", + "coverage.dat", "/coverage.dat", "/coverage.data", "/coverage1dat", @@ -55,6 +56,7 @@ public void optimizedMatchingPredicate( ".*?foo", ".*+foo", "^foo$", + "coverage\\.dat", ".*/coverage.dat", ".*/coverage\\.dat", ".*/test/.*/coverage\\.dat", diff --git a/src/test/java/com/google/devtools/common/options/BUILD b/src/test/java/com/google/devtools/common/options/BUILD index 598842cfbfc0a2..a67856fbb1bba3 100644 --- a/src/test/java/com/google/devtools/common/options/BUILD +++ b/src/test/java/com/google/devtools/common/options/BUILD @@ -43,6 +43,7 @@ java_test( deps = [ ":testutils", "//src/main/java/com/google/devtools/build/lib/util:classpath", + "//src/main/java/com/google/devtools/build/lib/util:string_encoding", "//src/main/java/com/google/devtools/common/options", "//src/main/java/com/google/devtools/common/options:invocation_policy", "//src/main/java/com/google/devtools/common/options/testing", diff --git a/src/test/java/com/google/devtools/common/options/RegexPatternConverterTest.java b/src/test/java/com/google/devtools/common/options/RegexPatternConverterTest.java index 8c6da91a7ecb9c..90c26c6b8a7967 100644 --- a/src/test/java/com/google/devtools/common/options/RegexPatternConverterTest.java +++ b/src/test/java/com/google/devtools/common/options/RegexPatternConverterTest.java @@ -16,6 +16,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; +import com.google.devtools.build.lib.util.StringEncoding; import com.google.devtools.common.options.Converters.RegexPatternConverter; import com.google.devtools.common.options.testing.ConverterTester; import com.google.testing.junit.testparameterinjector.TestParameterInjector; @@ -61,4 +62,39 @@ public void throwsForWrongPattern() { assertThrows(OptionsParsingException.class, () -> new RegexPatternConverter().convert("{")); assertThat(e).hasMessageThat().startsWith("Not a valid regular expression:"); } + + @Test + public void unicodeLiteral() throws OptionsParsingException { + // Options passed on the command line are passed to convertes in the internal encoding. + var regex = new RegexPatternConverter().convert(StringEncoding.unicodeToInternal("äöüÄÖÜß🌱")); + assertThat(regex.regexPattern().matcher("äöüÄÖÜß🌱").matches()).isTrue(); + assertThat(regex.matcher().test(StringEncoding.unicodeToInternal("äöüÄÖÜß🌱"))).isTrue(); + } + + @Test + public void unicodeLiteral_caseInsensitive() throws OptionsParsingException { + // Options passed on the command line are passed to convertes in the internal encoding. + var regex = + new RegexPatternConverter().convert(StringEncoding.unicodeToInternal("(?ui)äöüÄÖÜß🌱")); + assertThat(regex.regexPattern().matcher("ÄÖÜäöüß🌱").matches()).isTrue(); + assertThat(regex.matcher().test(StringEncoding.unicodeToInternal("ÄÖÜäöüß🌱"))).isTrue(); + } + + @Test + public void unicodeLiteral_suffix() throws OptionsParsingException { + // Options passed on the command line are passed to convertes in the internal encoding. + var regex = + new RegexPatternConverter().convert(StringEncoding.unicodeToInternal(".*äöüÄÖÜß🌱")); + assertThat(regex.regexPattern().matcher("äöüäöüÄÖÜß🌱").matches()).isTrue(); + assertThat(regex.matcher().test(StringEncoding.unicodeToInternal("äöüäöüÄÖÜß🌱"))).isTrue(); + } + + @Test + public void unicodeClass() throws OptionsParsingException { + // Options passed on the command line are passed to convertes in the internal encoding. + var regex = + new RegexPatternConverter().convert(StringEncoding.unicodeToInternal("\\p{L}{7}\\p{IsEmoji}")); + assertThat(regex.regexPattern().matcher("äöüÄÖÜß🌱").matches()).isTrue(); + assertThat(regex.matcher().test(StringEncoding.unicodeToInternal("äöüÄÖÜß🌱"))).isTrue(); + } } diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index be42ada133b340..1ad58282a8fb25 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -233,18 +233,14 @@ sh_test( srcs = ["bazel_java17_test.sh"], args = [ # java_tools zip to test - "$(rlocationpath //src:java_tools_java8.zip)", - "$(rlocationpath //src:java_tools_prebuilt_java8.zip)", + "$(rlocationpath //src:java_tools.zip)", + "$(rlocationpath //src:java_tools_prebuilt.zip)", ], data = [ ":gen_rules_java_repo_name", ":test-deps", - # Use java_tools compiled at Java language 8 instead (aligned with the - # default compilation level of java_tools). For the purpose - # of this test, we need java_tools to be running with runtime >11 to - # test the failure modes of incompatible system classpaths. - "//src:java_tools_prebuilt_java8.zip", - "//src:java_tools_java8.zip", + "//src:java_tools.zip", + "//src:java_tools_prebuilt.zip", "@bazel_tools//tools/bash/runfiles", ], tags = ["local"], @@ -294,8 +290,8 @@ JAVA_VERSIONS_COVERAGE = ("11", "17", "21") srcs = ["bazel_java_test.sh"], args = [ # java_tools zips to test - "$(rlocationpath %s)" % ("//src:java_tools_zip" if java_version == "21" else "//src:java_tools_java8.zip"), - "$(rlocationpath %s)" % ("//src:java_tools_prebuilt_zip" if java_version == "21" else "//src:java_tools_prebuilt_java8.zip"), + "$(rlocationpath //src:java_tools_zip)", + "$(rlocationpath //src:java_tools_prebuilt_zip)", # --java_language_version value java_version, # --java_runtime_version value @@ -304,8 +300,8 @@ JAVA_VERSIONS_COVERAGE = ("11", "17", "21") data = [ ":gen_rules_java_repo_name", ":test-deps", - "//src:java_tools_zip" if java_version == "21" else "//src:java_tools_java8.zip", - "//src:java_tools_prebuilt_zip" if java_version == "21" else "//src:java_tools_prebuilt_java8.zip", + "//src:java_tools_prebuilt_zip", + "//src:java_tools_zip", "@bazel_tools//tools/bash/runfiles", ], exec_compatible_with = ["//:highcpu_machine"], @@ -560,8 +556,8 @@ sh_test( srcs = ["bazel_coverage_java_test.sh"], args = [ # java_tools zips to test - "$(rlocationpath %s)" % ("//src:java_tools_zip" if java_version == "21" else "//src:java_tools_java8.zip"), - "$(rlocationpath %s)" % ("//src:java_tools_prebuilt_zip" if java_version == "21" else "//src:java_tools_prebuilt_java8.zip"), + "$(rlocationpath //src:java_tools_zip)", + "$(rlocationpath //src:java_tools_prebuilt_zip)", # WORKSPACE file of the coverage output generator repo to test "$(rlocationpath //tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_empty_workspace)", # --java_runtime_version value @@ -570,10 +566,10 @@ sh_test( data = [ ":gen_rules_java_repo_name", ":test-deps", - "//src:java_tools_prebuilt_zip" if java_version == "21" else "//src:java_tools_prebuilt_java8.zip", - "//src:java_tools_zip" if java_version == "21" else "//src:java_tools_java8.zip", + "//src:java_tools_prebuilt_zip", + "//src:java_tools_zip", "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_empty_workspace", - "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator_repo%s" % ("" if java_version == "21" else "_java8"), + "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator_repo", ], tags = ["no_windows"], ) @@ -628,7 +624,7 @@ sh_test( data = [ ":test-deps", "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_empty_workspace", - "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator_repo_java8", + "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator_repo", ], tags = [ "no_windows", diff --git a/src/upload_all_java_tools.sh b/src/upload_all_java_tools.sh index 9910ac31ff215e..59880df0982262 100755 --- a/src/upload_all_java_tools.sh +++ b/src/upload_all_java_tools.sh @@ -58,7 +58,7 @@ commit_hash=$(git rev-parse HEAD) timestamp=$(date +%s) bazel_version=$(bazel info release | cut -d' ' -f2) -RELEASE_BUILD_OPTS="-c opt --tool_java_language_version=8 --java_language_version=8" +RELEASE_BUILD_OPTS="-c opt" # Check that the build machine is set up for Unicode. bazel run ${RELEASE_BUILD_OPTS} //src:CheckSunJnuEncoding diff --git a/tools/build_defs.bzl b/tools/build_defs.bzl index d50c414fa31045..21e262a7104244 100644 --- a/tools/build_defs.bzl +++ b/tools/build_defs.bzl @@ -12,45 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Utility for compiling at Java 8.""" +"""Utilities for managing tools for different platforms.""" load("@platforms//host:constraints.bzl", "HOST_CONSTRAINTS") visibility("//tools/...") -_java_language_version_8_transition = transition( - implementation = lambda settings, attr: { - "//command_line_option:java_language_version": "8", - }, - inputs = [], - outputs = ["//command_line_option:java_language_version"], -) - -def _transition_java_language_8_files_impl(ctx): - return [ - DefaultInfo( - files = depset(ctx.files.files), - ), - ] - -_transitioned_java_8_files = rule( - implementation = _transition_java_language_8_files_impl, - attrs = { - "files": attr.label_list( - allow_files = True, - cfg = _java_language_version_8_transition, - mandatory = True, - ), - }, -) - -def transition_java_language_8_filegroup(name, files, visibility): - _transitioned_java_8_files( - name = name, - files = files, - visibility = visibility, - ) - BZLMOD_ENABLED = str(Label("@bazel_tools//:foo")).startswith("@@") IS_HOST_WINDOWS = Label("@platforms//os:windows") in [Label(label) for label in HOST_CONSTRAINTS] diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD index 15a7afb2295885..c0ce4cf2196700 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD @@ -1,7 +1,6 @@ -load("@rules_java//java:java_binary.bzl", "java_binary") load("@rules_java//java:java_library.bzl", "java_library") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load("//tools:build_defs.bzl", "transition_java_language_8_filegroup") +load("//src:release_archive.bzl", "minimum_java_runtime_java_binary") package( default_visibility = [ @@ -11,7 +10,7 @@ package( licenses(["notice"]) # Apache 2.0 -java_binary( +minimum_java_runtime_java_binary( name = "all_lcov_merger_tools", visibility = ["//visibility:public"], runtime_deps = [":lcov_merger_lib"], @@ -227,13 +226,3 @@ filegroup( ], visibility = ["//src/test/shell/bazel:__pkg__"], ) - -transition_java_language_8_filegroup( - name = "coverage_output_generator_repo_java8", - files = [ - ":coverage/BUILD", - ":coverage/WORKSPACE", - ":coverage/all_lcov_merger_tools_deploy.jar", - ], - visibility = ["//src/test/shell/bazel:__pkg__"], -) diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh index 6237aecc996547..1bbf515db8ff3c 100755 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/build_and_upload.sh @@ -24,10 +24,7 @@ commit_hash=$(git rev-parse HEAD) timestamp=$(date +%s) bazel_version=$(bazel info release | cut -d' ' -f2) -bazel build \ - --java_language_version=8 \ - --tool_java_language_version=8 \ - //tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator.zip +bazel build -c opt //tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator.zip cov_gen_zip="bazel-bin/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/coverage_output_generator.zip" @@ -46,7 +43,7 @@ To build the same zip from source, run the commands: $ git clone https://github.com/bazelbuild/bazel.git $ git checkout ${commit_hash} -$ bazel build --java_language_version=8 --tool_java_language_version=8 //tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator.zip +$ bazel build -c opt //tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator.zip EOF zip -rjv "${tmp_zip}" "${readme_file}"