Skip to content

Commit 4c8d0a0

Browse files
nedtwiggnedtwigg
authored
Huge speedup by separating "find" from "generate" (#66)
2 parents d691de0 + 69f9e10 commit 4c8d0a0

File tree

8 files changed

+170
-93
lines changed

8 files changed

+170
-93
lines changed

‎CHANGELOG.md

+3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# AtPlug releases
22

33
## [Unreleased]
4+
### Added
5+
- Cacheability, incremental task, and huge speed improvement. ([#66](https://github.com/diffplug/atplug/pull/66))
6+
- `plugGenerate` refactored into `plugFind` followed by `plugGenerate`
47
### Changed
58
- Bump required JVM from 11 to 17. ([#63](https://github.com/diffplug/atplug/pull/63))
69
- Detect Kotlin version rather than harcode it. ([#64](https://github.com/diffplug/atplug/pull/64))

‎atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/PlugGenerator.kt

+19-35
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,18 @@ import java.lang.UnsatisfiedLinkError
2222
import java.lang.reflect.Constructor
2323
import java.lang.reflect.Modifier
2424
import java.net.URLClassLoader
25-
import java.nio.file.FileVisitResult
26-
import java.nio.file.Files
27-
import java.nio.file.Path
28-
import java.nio.file.SimpleFileVisitor
29-
import java.nio.file.attribute.BasicFileAttributes
3025
import java.util.*
3126
import java.util.function.Function
3227
import kotlin.reflect.KFunction
3328
import kotlin.reflect.full.companionObjectInstance
3429
import kotlin.reflect.full.memberFunctions
3530

36-
class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst: Set<File>) {
31+
class PlugGenerator
32+
internal constructor(
33+
plugToSocket: Map<String, String>,
34+
toSearches: List<File>,
35+
toLinkAgainst: Set<File>
36+
) {
3737
@JvmField val atplugInf: SortedMap<String, String> = TreeMap()
3838

3939
/** A cache from a plugin interface to a function that converts a class into its metadata. */
@@ -56,22 +56,7 @@ class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst:
5656
}!!
5757
as KFunction<Function<Any, String>>
5858
try {
59-
val parser = PlugParser()
60-
// walk toSearch, passing each classfile to load()
61-
for (toSearch in toSearches) {
62-
if (toSearch.isDirectory) {
63-
Files.walkFileTree(
64-
toSearch.toPath(),
65-
object : SimpleFileVisitor<Path>() {
66-
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
67-
if (file.toString().endsWith(EXT_CLASS)) {
68-
maybeGeneratePlugin(parser, file)
69-
}
70-
return FileVisitResult.CONTINUE
71-
}
72-
})
73-
}
74-
}
59+
plugToSocket.forEach { (plug, socket) -> atplugInf[plug] = generatePlugin(plug, socket) }
7560
} finally {
7661
classLoader.close()
7762
System.setProperty("atplug.generate", "")
@@ -82,18 +67,13 @@ class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst:
8267
* Loads a class by its FQN. If it's concrete and implements a plugin, then we'll call
8368
* generatePlugin.
8469
*/
85-
private fun maybeGeneratePlugin(parser: PlugParser, path: Path) {
86-
parser.parse(path.toFile())
87-
if (!parser.hasPlug()) {
88-
return
89-
}
90-
val plugClass = classLoader.loadClass(parser.plugClassName)
91-
val socketClass = classLoader.loadClass(parser.socketClassName)
70+
private fun generatePlugin(plugClassName: String, socketClassName: String): String {
71+
val plugClass = classLoader.loadClass(plugClassName)
72+
val socketClass = classLoader.loadClass(socketClassName)
9273
require(!Modifier.isAbstract(plugClass.modifiers)) {
9374
"Class $plugClass has @Plug($socketClass) but it is abstract."
9475
}
95-
val atplugInfContent = generatePlugin<Any, Any>(plugClass, socketClass)
96-
atplugInf[plugClass.name] = atplugInfContent
76+
return generatePlugin<Any, Any>(plugClass, socketClass)
9777
}
9878

9979
private fun <SocketT, PlugT : SocketT> generatePlugin(
@@ -128,14 +108,18 @@ class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst:
128108
* Returns a Map from a plugin's name to its ATPLUG-INF content.
129109
*
130110
* @param toSearch a directory containing class files where we will look for plugin
131-
* implementations
111+
* implementations
132112
* @param toLinkAgainst the classes that these plugins implementations need
133113
* @return a map from component name to is ATPLUG-INF string content
134114
*/
135-
fun generate(toSearch: List<File>, toLinkAgainst: Set<File>): SortedMap<String, String> {
115+
fun generate(
116+
plugToSocket: Map<String, String>,
117+
toSearch: List<File>,
118+
toLinkAgainst: Set<File>
119+
): SortedMap<String, String> {
136120
return try {
137-
val ext = PlugGeneratorJavaExecable(toSearch, toLinkAgainst)
138-
val metadataGen = PlugGenerator(ext.toSearch, ext.toLinkAgainst)
121+
val ext = PlugGeneratorJavaExecable(plugToSocket, toSearch, toLinkAgainst)
122+
val metadataGen = PlugGenerator(plugToSocket, ext.toSearch, ext.toLinkAgainst)
139123
// save our results, with no reference to the guts of what happened inside PluginMetadataGen
140124
metadataGen.atplugInf
141125
} catch (e: Exception) {

‎atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/PlugGeneratorJavaExecable.kt

+9-9
Original file line numberDiff line numberDiff line change
@@ -20,21 +20,21 @@ import java.io.File
2020
import java.util.*
2121

2222
/** [PlugGenerator.PlugGenerator] in a [JavaExecable] form. */
23-
class PlugGeneratorJavaExecable(toSearch: List<File>?, toLinkAgainst: Set<File>?) : JavaExecable {
23+
class PlugGeneratorJavaExecable(
24+
plugToSocket: Map<String, String>,
25+
toSearch: List<File>,
26+
toLinkAgainst: Set<File>
27+
) : JavaExecable {
2428
// inputs
25-
var toSearch: List<File>
26-
var toLinkAgainst: Set<File>
29+
var plugToSocket: Map<String, String> = LinkedHashMap(plugToSocket)
30+
var toSearch: List<File> = ArrayList(toSearch)
31+
var toLinkAgainst: Set<File> = LinkedHashSet(toLinkAgainst)
2732

2833
// outputs
2934
@JvmField var atplugInf: SortedMap<String, String>? = null
3035

31-
init {
32-
this.toSearch = ArrayList(toSearch)
33-
this.toLinkAgainst = LinkedHashSet(toLinkAgainst)
34-
}
35-
3636
override fun run() {
37-
val metadataGen = PlugGenerator(toSearch, toLinkAgainst)
37+
val metadataGen = PlugGenerator(plugToSocket, toSearch, toLinkAgainst)
3838
atplugInf = metadataGen.atplugInf
3939
}
4040
}

‎atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/PlugParser.kt

+8-9
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import org.objectweb.asm.Type
2626
class PlugParser {
2727
private var buffer = ByteArray(64 * 1024)
2828

29-
fun parse(file: File) {
29+
fun parse(file: File): Pair<String, String>? {
3030
val filelen = (file.length() + 1).toInt() // +1 prevents infinite loop
3131
if (buffer.size < filelen) {
3232
buffer = ByteArray(filelen)
@@ -42,18 +42,17 @@ class PlugParser {
4242
}
4343
}
4444
val reader = ClassReader(buffer, 0, pos)
45-
plugClassName = asmToJava(reader.className)
45+
val plugClassName = asmToJava(reader.className)
46+
this.plugClassName = plugClassName
4647
socketClassName = null
48+
// set socketClassName if there is an `@Plug`
4749
reader.accept(
4850
classVisitor, ClassReader.SKIP_FRAMES or ClassReader.SKIP_DEBUG or ClassReader.SKIP_CODE)
51+
return socketClassName?.let { plugClassName to it }
4952
}
5053

51-
fun hasPlug(): Boolean {
52-
return socketClassName != null
53-
}
54-
55-
var plugClassName: String? = null
56-
var socketClassName: String? = null
54+
private var plugClassName: String? = null
55+
private var socketClassName: String? = null
5756

5857
private val classVisitor: ClassVisitor =
5958
object : ClassVisitor(API) {
@@ -73,7 +72,7 @@ class PlugParser {
7372
}
7473

7574
companion object {
76-
fun asmToJava(className: String): String {
75+
private fun asmToJava(className: String): String {
7776
return className.replace("/", ".")
7877
}
7978

‎atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/gradle/FindPlugsTask.kt

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package com.diffplug.atplug.tooling.gradle
2+
3+
import com.diffplug.atplug.tooling.PlugParser
4+
import java.io.File
5+
import org.gradle.api.DefaultTask
6+
import org.gradle.api.file.ConfigurableFileCollection
7+
import org.gradle.api.file.DirectoryProperty
8+
import org.gradle.api.tasks.*
9+
import org.gradle.work.*
10+
11+
/**
12+
* Incrementally scans compiled classes for @Plug usage and writes discovered plug classes into an
13+
* output directory.
14+
*/
15+
@CacheableTask
16+
abstract class FindPlugsTask : DefaultTask() {
17+
@get:CompileClasspath
18+
@get:Incremental
19+
@get:InputFiles
20+
abstract val classesFolders: ConfigurableFileCollection
21+
22+
/** Directory where we will store discovered plugs. */
23+
@get:OutputDirectory abstract val discoveredPlugsDir: DirectoryProperty
24+
25+
@TaskAction
26+
fun findPlugs(inputChanges: InputChanges) {
27+
// If not incremental, clear everything and rescan
28+
if (!inputChanges.isIncremental) {
29+
discoveredPlugsDir.get().asFile.deleteRecursively()
30+
}
31+
32+
// Make sure our output directory exists
33+
discoveredPlugsDir.get().asFile.mkdirs()
34+
35+
// For each changed file in classesFolders, determine if it has @Plug
36+
val parser = PlugParser()
37+
for (change in inputChanges.getFileChanges(classesFolders)) {
38+
if (!change.file.name.endsWith(".class")) {
39+
continue
40+
}
41+
when (change.changeType) {
42+
ChangeType.REMOVED -> {
43+
// Remove old discovered data for this file
44+
removeOldMetadata(change.file)
45+
}
46+
ChangeType.ADDED,
47+
ChangeType.MODIFIED -> {
48+
parseAndWriteMetadata(parser, change.file)
49+
}
50+
}
51+
}
52+
}
53+
54+
private fun parseAndWriteMetadata(parser: PlugParser, classFile: File) {
55+
val plugToSocket = parser.parse(classFile)
56+
if (plugToSocket != null) {
57+
// For example: write a single line containing the discovered plug FQN
58+
val discoveredFile = discoveredPlugsDir.file(classFile.nameWithoutExtension).get().asFile
59+
if (discoveredFile.exists()) {
60+
val existing = discoveredFile.readText().split("|")
61+
check(existing[0] == plugToSocket.first) {
62+
"You need to rename one of these plugs because they have the same classfile name: ${existing[0]} and $plugToSocket"
63+
}
64+
} else {
65+
discoveredFile.parentFile.mkdirs()
66+
}
67+
discoveredFile.writeText(plugToSocket.let { "${it.first}|${it.second}" })
68+
} else {
69+
// If previously discovered, remove it
70+
removeOldMetadata(classFile)
71+
}
72+
}
73+
74+
private fun removeOldMetadata(classFile: File) {
75+
// Remove any discovered file for the old .class
76+
val possibleName = classFile.nameWithoutExtension
77+
val discoveredFile = discoveredPlugsDir.file(possibleName).get().asFile
78+
if (discoveredFile.exists()) {
79+
discoveredFile.delete()
80+
}
81+
}
82+
}

‎atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/gradle/PlugGenerateTask.kt

+24-24
Original file line numberDiff line numberDiff line change
@@ -30,20 +30,15 @@ import java.util.jar.Manifest
3030
import javax.inject.Inject
3131
import org.gradle.api.DefaultTask
3232
import org.gradle.api.file.ConfigurableFileCollection
33+
import org.gradle.api.file.DirectoryProperty
3334
import org.gradle.api.file.FileCollection
3435
import org.gradle.api.plugins.JavaPluginExtension
3536
import org.gradle.api.provider.Property
36-
import org.gradle.api.tasks.Classpath
37-
import org.gradle.api.tasks.InputFiles
38-
import org.gradle.api.tasks.Internal
39-
import org.gradle.api.tasks.JavaExec
40-
import org.gradle.api.tasks.Nested
41-
import org.gradle.api.tasks.Optional
42-
import org.gradle.api.tasks.OutputDirectory
43-
import org.gradle.api.tasks.TaskAction
37+
import org.gradle.api.tasks.*
4438
import org.gradle.jvm.toolchain.JavaLauncher
4539
import org.gradle.jvm.toolchain.JavaToolchainService
4640
import org.gradle.process.JavaForkOptions
41+
import org.gradle.work.Incremental
4742
import org.gradle.workers.ProcessWorkerSpec
4843
import org.gradle.workers.WorkerExecutor
4944

@@ -52,6 +47,8 @@ abstract class PlugGenerateTask : DefaultTask() {
5247

5348
@get:Inject abstract val workerExecutor: WorkerExecutor
5449

50+
@get:InputDirectory abstract val discoveredPlugsDir: DirectoryProperty
51+
5552
@get:InputFiles @get:Classpath abstract val jarsToLinkAgainst: ConfigurableFileCollection
5653

5754
@get:Internal var resourcesFolder: File? = null
@@ -60,7 +57,7 @@ abstract class PlugGenerateTask : DefaultTask() {
6057
val atplugInfFolder: File
6158
get() = File(resourcesFolder, PlugPlugin.ATPLUG_INF)
6259

63-
@InputFiles var classesFolders: FileCollection? = null
60+
@get:Incremental @get:InputFiles abstract val classesFolders: ConfigurableFileCollection
6461

6562
init {
6663
this.outputs.upToDateWhen {
@@ -74,20 +71,22 @@ abstract class PlugGenerateTask : DefaultTask() {
7471
launcher.set(service.launcherFor(spec))
7572
}
7673

77-
fun setClassesFolders(files: Iterable<File>) {
78-
// if we don't copy, Gradle finds an implicit dependency which
79-
// forces us to depend on `classes` even though we don't
80-
val copy: MutableList<File> = ArrayList()
81-
for (file in files) {
82-
copy.add(file)
83-
}
84-
classesFolders = project.files(copy)
85-
}
86-
8774
@TaskAction
8875
fun build() {
76+
val discoveredFiles = discoveredPlugsDir.get().asFile.listFiles().orEmpty().filter { it.isFile }
77+
val plugsToSockets =
78+
discoveredFiles.associate {
79+
val pieces = it.readText().split("|")
80+
pieces[0] to pieces[1]
81+
}
82+
if (plugsToSockets.isEmpty()) {
83+
// no discovered plugs
84+
FileMisc.cleanDir(atplugInfFolder)
85+
return
86+
}
87+
8988
// generate the metadata
90-
val result = generate()
89+
val result = generate(plugsToSockets)
9190

9291
// clean out the ATPLUG-INF folder, and put the map's content into the folder
9392
FileMisc.cleanDir(atplugInfFolder)
@@ -97,8 +96,8 @@ abstract class PlugGenerateTask : DefaultTask() {
9796
}
9897

9998
// the resources directory *needs* the Service-Component entry of the manifest to exist in order
100-
// for tests to work
101-
// so we'll get a manifest (empty if necessary, but preferably we'll load what already exists)
99+
// for tests to work so we'll get a manifest (empty if necessary, but preferably we'll load what
100+
// already exists)
102101
val manifest = loadManifest()
103102
val componentsCmd = atplugComponents(atplugInfFolder)
104103
val componentsActual = manifest.mainAttributes.getValue(PlugPlugin.SERVICE_COMPONENT)
@@ -138,9 +137,10 @@ abstract class PlugGenerateTask : DefaultTask() {
138137
}
139138
}
140139

141-
private fun generate(): SortedMap<String, String> {
140+
private fun generate(discoveredPlugs: Map<String, String>): SortedMap<String, String> {
142141
val input =
143-
PlugGeneratorJavaExecable(ArrayList(classesFolders!!.files), jarsToLinkAgainst.files)
142+
PlugGeneratorJavaExecable(
143+
discoveredPlugs, ArrayList(classesFolders.files), jarsToLinkAgainst.files)
144144
return if (launcher.isPresent) {
145145
val workQueue =
146146
workerExecutor.processIsolation { workerSpec: ProcessWorkerSpec ->

0 commit comments

Comments
 (0)