fix(web-compliance): include missing dependencies in (manual) NOTICE.html (#45)

#### Description of changes

In an earlier PR, we added a task to have Component Governance automatically generate NOTICE.html files for us. Unfortunately, there are several problems that block this approach from being accurate:

1. We have some cases where we use code that Android Studio generated, which means we need an attribution for The Android Open Source Project which Gradle/Component Governance doesn't know about
2. Component Governance will only include "supported components" in NOTICE files, which excludes anything from a cgmanifest file and notably excludes `androidx.annotation`, which is a Maven dependency not present in Maven Central
3. Component Governance doesn't have a deterministic way to tell whether a given dependency is a devDependency or not, so it is conservative and always assumes "real dependency". Unfortunately, this means that every time a devDependency is bumped by dependabot, we need to remember to go manually configure CG to omit the new version, or it will start showing up in NOTICE files

We can sort of work around 1 with a cgmanifest file, but it immediately runs into 2. We don't have workarounds for 2 and 3. Until we do, we have to go back to manually curating NOTICE.html.

This PR does that, but extends and automates the use of the `list-release-dependencies` script into something that verifies that the lockfile and NOTICE.html are in sync as a condition of PR builds passing, to avoid the case where Dependabot updates a version of something important and we forget to update NOTICE.html.

It also adds the necessary copyright header for AOSP to the file in question that triggers us to include it.

#### Pull request checklist

<!-- If a checklist item is not applicable to this change, write "n/a" in the checkbox -->

- [n/a] Addresses an existing issue: #0000
- [n/a] Added/updated relevant unit test(s)
- [x] Ran `./gradlew fastpass` from `AccessibilityInsightsForAndroidService`
- [x] PR title _AND_ final merge commit title both start with a semantic tag (`fix:`, `chore:`, `feat(feature-name):`, `refactor:`).

Author: dbjorge

AccessibilityInsightsForAndroidService/app/gradle.lockfile

@@ -10,6 +10,7 @@ com.android.tools.analytics-library:protos:27.0.1=lintClassPath
 com.android.tools.analytics-library:shared:27.0.1=lintClassPath
 com.android.tools.analytics-library:tracker:27.0.1=lintClassPath
 com.android.tools.build:aapt2-proto:0.4.0=lintClassPath
+com.android.tools.build:aapt2:4.0.1-6197926=_internal_aapt2_binary
 com.android.tools.build:apksig:4.0.1=lintClassPath
 com.android.tools.build:apkzlib:4.0.1=lintClassPath
 com.android.tools.build:builder-model:4.0.1=lintClassPath

AccessibilityInsightsForAndroidService/app/src/main/java/com/microsoft/accessibilityinsightsforandroidservice/AccessibilityNodeInfoSorter.java

@@ -1,5 +1,19 @@
-// Copyright (c) Microsoft Corporation.
+// Portions Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
+//
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package com.microsoft.accessibilityinsightsforandroidservice;
 

AccessibilityInsightsForAndroidService/build.gradle

@@ -38,8 +38,8 @@ spotless {
     }
     java {
         target '**/*.java'
-        // This file has the APACHE license header, which we need to keep
-        targetExclude '**/AccessibilityInsightsForAndroidService.java'
+        // These files have an APACHE license header, which we need to keep
+        targetExclude '**/AccessibilityInsightsForAndroidService.java','**/AccessibilityNodeInfoSorter.java'
         licenseHeader '// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\n'
     }
 }

AccessibilityInsightsForAndroidService/list-release-dependencies.js

@@ -101,25 +101,22 @@ jobs:
 
       # terms artifact
 
+      - task: ComponentGovernanceComponentDetection@0
+        displayName: '(terms) dependency detection (Component Governance)'
+        timeoutInMinutes: 5
+
+      - script: node $(system.defaultWorkingDirectory)/pipeline/verify-notice-contents.js
+        displayName: (terms) verify NOTICE.html content matches lockfile
+
       - task: CopyFiles@2
-        displayName: (terms) copy LICENSE to artifact staging
+        displayName: (terms) copy LICENSE and NOTICE.html to artifact staging
         inputs:
           contents: |
             LICENSE
+            NOTICE.html
           sourceFolder: '$(system.defaultWorkingDirectory)'
           targetFolder: '$(build.artifactstagingdirectory)/terms'
 
-      - task: ComponentGovernanceComponentDetection@0
-        displayName: '(terms) dependency detection (Component Governance)'
-        timeoutInMinutes: 5
-
-      - task: msospo.ospo-extension.8d7f9abb-6896-461d-9e25-4f74ed65ddb2.notice@0
-        displayName: '(terms) generate NOTICE.html file'
-        inputs:
-          outputfile: '$(build.artifactstagingdirectory)/terms/NOTICE.html'
-          outputformat: html
-        timeoutInMinutes: 5
-
       - task: PublishPipelineArtifact@1
         displayName: (terms) publish artifact
         inputs:

NOTICE.html

@@ -2,25 +2,28 @@
 # Licensed under the MIT License.
 
 trigger:
-    - master
+  - master
 
 jobs:
-    - job: 'gradlew_build'
+  - job: 'gradlew_build'
 
-      pool:
-          vmImage: 'windows-latest'
+    pool:
+        vmImage: 'windows-latest'
 
-      steps:
-          - task: Gradle@2
-            displayName: build and test dev ai-android service
-            inputs:
-              workingDirectory: '$(system.defaultWorkingDirectory)/AccessibilityInsightsForAndroidService'
-              gradleWrapperFile: 'AccessibilityInsightsForAndroidService/gradlew'
-              gradleOptions: '-Xmx3072m'
-              options: -S
-              javaHomeOption: 'JDKVersion'
-              jdkVersionOption: '1.8'
-              jdkArchitectureOption: 'x64'
-              publishJUnitResults: true
-              testResultsFiles: '**/TEST-*.xml'
-              tasks: 'build'
+    steps:
+      - task: Gradle@2
+        displayName: build and test dev ai-android service
+        inputs:
+          workingDirectory: '$(system.defaultWorkingDirectory)/AccessibilityInsightsForAndroidService'
+          gradleWrapperFile: 'AccessibilityInsightsForAndroidService/gradlew'
+          gradleOptions: '-Xmx3072m'
+          options: -S
+          javaHomeOption: 'JDKVersion'
+          jdkVersionOption: '1.8'
+          jdkArchitectureOption: 'x64'
+          publishJUnitResults: true
+          testResultsFiles: '**/TEST-*.xml'
+          tasks: 'build'
+
+      - script: node $(system.defaultWorkingDirectory)/pipeline/verify-notice-contents.js
+        displayName: verify NOTICE.html content matches lockfile

pipeline/release-build.yaml

@@ -0,0 +1,106 @@
+// USAGE:
+//
+//     node verify-notice-contents.js
+//
+// Verifies that gradle.lockfile and NOTICE.html are in sync. Exits with non-zero if NOTICE.html
+// contains any listings that gradle.lockfile doesn't suggest are required OR if NOTICE.html is
+// missing any listings that gradle.lockfile suggests are required.
+
+const fs = require('fs');
+const path = require('path');
+const process = require('process');
+
+const lockfilePath = path.join(__dirname, '..', 'AccessibilityInsightsForAndroidService', 'app', 'gradle.lockfile');
+const noticePath = path.join(__dirname, '..', 'NOTICE.html')
+
+// This is to cover cases where we've used code generated by Android Studio (equals/hashCode implementations, for example)
+const releaseDepsNotKnownToGradle = [
+    'The Android Open Source Project'
+]
+
+function isDevTarget(target) {
+    return target === 'lintClassPath' || target === '_internal_aapt2_binary' || /Test/.test(target);
+}
+function isReleaseTarget(target) {
+    return !isDevTarget(target);
+}
+function isEmptyLine(line) {
+    // whitespace until EOL or start of comment
+    return /^\s*(\#|$)/.test(line);
+}
+
+// input format: lines like "com.android.tools.build:apkzlib:4.0.1=target1,target2"
+// output format: ["com.android.tools.build/apkzlib 4.0.1", ...]
+function parseReleaseDepsFromLockfile(path) {
+    const lockfileContent = fs.readFileSync(path).toString();
+    const lockfileLines = lockfileContent.split(/\r?\n/);
+    const output = [];
+    for (const line of lockfileLines) {
+        if (isEmptyLine(line) || line.startsWith('empty=')) {
+            continue;
+        }
+    
+        const parts = line.split('=');
+        if (parts.length !== 2) {
+            throw new Error(`malformatted line: ${line}`);
+        }
+    
+        const [dep, targetsLine] = parts;
+        const targets = targetsLine.split(',');
+    
+        if (targets.some(isReleaseTarget)) {
+            const [groupId, artifactId, version] = dep.split(':');
+            noticeFormatDep = `${groupId}/${artifactId} ${version}`;
+            output.push(noticeFormatDep);
+        }
+    }
+    output.sort();
+    return output;
+}
+
+// input format: HTML page with snippets per dep like
+//
+// <summary>
+//   org.jetbrains/annotations 15.0 - Apache-2.0
+// </summary>
+//
+// output format: ["org.jetbrains/annotatoins 15.0", ...]
+function parseDepsFromNoticeFile(path) {
+    const noticeContent = fs.readFileSync(path).toString();
+    const componentRegex = /\<summary\>\s*([a-zA-Z0-9\._\- \/]+) - ([a-zA-Z0-9\._\-]+)\s*\<\/summary\>/gm
+    let captureGroups;
+    const deps = [];
+    while ((captureGroups = componentRegex.exec(noticeContent)) !== null) {
+        deps.push(captureGroups[1]);
+    }
+    deps.sort();
+    return deps;
+}
+
+function difference(from, to) {
+    return from.filter(candidate => !to.includes(candidate));
+}
+
+const releaseDeps = parseReleaseDepsFromLockfile(lockfilePath);
+releaseDeps.push(...releaseDepsNotKnownToGradle);
+
+const noticeDeps = parseDepsFromNoticeFile(noticePath);
+
+const missingDeps = difference(releaseDeps, noticeDeps);
+const extraneousDeps = difference(noticeDeps, releaseDeps);
+
+let exitCode = 0;
+
+if (extraneousDeps.length > 0) {
+    console.error(`Error: extraneous deps found in ${noticePath} which aren't required by ${lockfilePath}:`);
+    extraneousDeps.forEach(dep => console.error(`  "${dep}"`));
+    exitCode = 1;
+}
+if (missingDeps.length > 0) {
+    console.error(`Error: deps found in ${lockfilePath} but not ${noticePath}:`);
+    missingDeps.forEach(dep => console.error(`  "${dep}"`));
+    console.error(`If these deps were recently added/updated (eg, this is a Dependabot PR), you will need to regenerate this NOTICE file and commit it to the PR`);
+    exitCode = 2;
+}
+
+process.exit(exitCode);