Create batch_image_correlation.yml

Author: gbrencher

.github/workflows/batch_image_correlation.yml

@@ -0,0 +1,149 @@
+# Batch process many image pairs
+name: Batch
+run-name: batch image correlation ${{ inputs.npairs }} connections
+
+on:
+  workflow_dispatch:
+    inputs:
+        cloud_cover:
+            type: choice
+            required: true
+            description: percent cloud cover allowed in images
+            default: '10'
+        start_month:
+            type: choice
+            required: true
+            description: first month of year to search for images
+            default: '6'
+            options: ['1','2','3','4','5','6','7','8','9','10','11','12']
+        stop_month:
+            type: choice
+            required: true
+            description: last month of year to search for images
+            default: '9'
+            options: ['1','2','3','4','5','6','7','8','9','10','11','12']
+        npairs:
+            type: choice
+            required: true
+            description: number of pairs per image
+            default: '3'
+            options: ['3','2','1']
+
+  # Must duplicate inputs for workflow_call (https://github.com/orgs/community/discussions/39357)
+  workflow_call:
+    inputs:
+      cloud_cover:
+        type: string
+        required: true
+      start_month:
+        type: string
+        required: true
+      stop_month:
+        type: string
+        required: true
+      npairs:
+        type: string
+        required: true
+
+jobs:
+  # The output of this job is a JSON mapping for a matrix job
+  S2_search:
+    runs-on: ubuntu-latest
+    outputs:
+      BURST_IDS: ${{ steps.asf-search.outputs.BURST_IDS }}
+      MATRIX: ${{ steps.asf-search.outputs.MATRIX_PARAMS_COMBINATIONS }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Conda environment with Micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          cache-environment: true
+          environment-file: environment.yml
+
+      # https://words.yuvi.in/post/python-in-github-actions/
+      - name: Search aws for S2 imagery
+        id: S2_search
+        shell:  bash -el -c "python -u {0}"
+        run: | 
+          import xarray as xr
+          import os
+          import pystac
+          import pystac_client
+          import stackstac
+          from dask.distributed import Client
+          import dask
+          import json
+
+          # GDAL environment variables for better performance
+          os.environ['AWS_REGION']='us-west-2'
+          os.environ['GDAL_DISABLE_READDIR_ON_OPEN']='EMPTY_DIR' 
+          os.environ['AWS_NO_SIGN_REQUEST']='YES'
+
+          # hardcode bbox for now
+          bbox = {
+          "type": "Polygon",
+          "coordinates": [
+          [[75.42382800808971,36.41082887114753],
+          [75.19442677164156,36.41082887114753],
+          [75.19442677164156,36.201076360872946],
+          [75.42382800808971,36.201076360872946],
+          [75.42382800808971,36.41082887114753]]]
+          }
+
+          # We use the api from element84 to query the data
+          URL = "https://earth-search.aws.element84.com/v1"
+          catalog = pystac_client.Client.open(URL)
+
+          search = catalog.search(
+          collections=["sentinel-2-l2a"],
+          intersects=bbox,
+          query={"eo:cloud_cover": {"lt": ${{ inputs.cloud_cover }}}}, # less than 10% cloud cover
+          )
+
+          # Check how many items were returned
+          items = search.item_collection()
+          print(f"Returned {len(items)} Items")
+
+          sentinel2_stack = stackstac.stack(items)
+          sentinel2_stack_snowoff = sentinel2_stack.where((sentinel2_stack.time.dt.month >= ${{ inputs.start_month }}) & (sentinel2_stack.time.dt.month <= ${{ inputs.stop_month }}), drop=True)
+
+          image_dates = sentinel2_stack_snowoff.time.dt.strftime('%Y-%m-%d').values.tolist()
+          print('\n'.join(image_dates))
+
+          # Create Matrix Job Mapping (JSON Array)
+          pairs = []
+          for r in range(len(image_dates) - ${{ inputs.npairs }}):
+              for s in range(1, ${{ inputs.npairs }} + 1 ):
+                img1_date = image_dates[r]
+                img2_date = image_dates[r+s]
+                shortname = f'{img1_date}_{img2_date}'
+                pairs.append({'img1_date': img1_date, 'img2_date': img2_date, 'name':shortname})
+          matrixJSON = f'{{"include":{json.dumps(pairs)}}}'
+          print(f'Number of Interferograms: {len(pairs)}')
+          
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+              print(f'BURST_IDS={image_dates}', file=f)
+              print(f'MATRIX_PARAMS_COMBINATIONS={matrixJSON}', file=f)
+
+          #STOPPED HERE 5/29/2024
+
+  # A matrix job that calls a reuseable workflow
+  hyp3-isce2:
+    needs: searchASF
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.searchASF.outputs.MATRIX) }}
+    name: ${{ matrix.name }}
+    uses: ./.github/workflows/single-job.yml
+    with:
+      reference: ${{ matrix.reference }}
+      secondary: ${{ matrix.secondary }} 
+      looks: ${{ inputs.looks }}
+      apply_water_mask: ${{ inputs.apply_water_mask }}
+      workflow_name: ${{ matrix.name }}
+    secrets: inherit