Separate weekly openApi-doc refresh and command-metadata refresh

.azure-pipelines/command-metadata-refresh.yml

@@ -0,0 +1,150 @@
+# This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
+# Please make sure to check all the converted content, it is your team's responsibility to make sure that the pipeline is still valid and functions as expected.
+# This pipeline will be extended to the OneESPT template
+# If you are not using the E+D shared hosted pool with windows-2022, replace the pool section with your hosted pool, os, and image name. If you are using a Linux image, you must specify an additional windows image for SDL: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sdlanalysis/overview#how-to-specify-a-windows-pool-for-the-sdl-source-analysis-stage
+# The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish Module Artifacts' in the templateContext section.
+# The Task 'NuGetCommand@2' has been converted to an output named 'Publish NuGet to feed' in the templateContext section. 
+
+name: $(BuildDefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)
+parameters:
+- name: BuildAgent
+  displayName: Build Agent
+  default: 1es-windows-ps-compute-m
+- name: BaseBranch
+  displayName: Base Branch
+  default: main
+- name: Test
+  type: boolean
+  default: true
+- name: Pack
+  type: boolean
+  default: false
+- name: Sign
+  type: boolean
+  default: false
+- name: BumpModuleVersion
+  type: boolean
+  default: false
+- name: CreatePullRequest
+  type: boolean
+  default: true
+
+variables:
+  BuildAgent: ${{ parameters.BuildAgent }}
+  Branch: "ModuleCommandMetadataRefresh"
+  BaseBranch: ${{ parameters.BaseBranch }}
+
+trigger:
+  branches:
+    include:
+    - main
+
+resources:
+  repositories:
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool: $(BuildAgent)
+    sdl:
+      binskim:
+        enabled: false
+        justificationForDisabling: "Binskim keeps on crashing and failing the weekly build pipeline. Disabling it for now because we are unable to publish the artifacts to internal feeds."
+      credscan:
+        suppressionsFile: $(Build.SourcesDirectory)/.azure-pipelines/config/credscan/credscan-suppressions.json
+      policheck:
+        exclusionFile: $(Build.SourcesDirectory)/.azure-pipelines/config/policheck/policheck-exclusions.xml
+    customBuildTags:
+    - ES365AIMigrationTooling
+    stages:
+    - stage: stage
+      jobs:
+      - job: CreateMetadataRefreshBranch
+        displayName: Create Metadata Refresh Branch
+        timeoutInMinutes: 1200
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: ${{ parameters.BaseBranch }}
+        - task: PowerShell@2
+          name: "ComputeBranch"
+          displayName: "Compute Module Command Metadata Refresh Branch Name"
+          inputs:
+            targetType: inline
+            script: |
+              $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm)
+              Write-Host "##vso[task.setvariable variable=RefreshBranch;isOutput=true]$branch" 
+        - task: Bash@3
+          displayName: "Create Module Command Metadata Refresh Branch"
+          inputs:
+            targetType: inline
+            script: |
+              git status
+              git branch $(ComputeBranch.RefreshBranch)
+              git checkout $(ComputeBranch.RefreshBranch)
+              git status               
+
+      - job: MsGraphPsSdkModuleCommandMetadataGeneration
+        dependsOn: CreateMetadataRefreshBranch
+        displayName: Microsoft Graph PowerShell SDK Generation
+        condition: succeeded()
+        timeoutInMinutes: 840
+        variables:
+          RefreshBranch: $[ dependencies.CreateMetadataRefreshBranch.outputs['ComputeBranch.RefreshBranch'] ]
+        steps:
+        - template: .azure-pipelines/common-templates/checkout.yml@self
+          parameters:
+            TargetBranch: $(RefreshBranch)
+        - template: .azure-pipelines/common-templates/install-tools.yml@self
+        - template: .azure-pipelines/common-templates/security-pre-checks.yml@self
+        - template: .azure-pipelines/generation-templates/authentication-module.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/workload-modules.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/meta-module.yml@self
+          parameters:
+            Test: ${{ parameters.Test }}
+            Pack: ${{ parameters.Pack }}
+            Sign: ${{ parameters.Sign }}
+        - template: .azure-pipelines/generation-templates/generate-command-metadata.yml@self
+        - template: .azure-pipelines/common-templates/security-post-checks.yml@self
+
+        - ${{ if eq(parameters.BumpModuleVersion, true) }}:  
+          - task: PowerShell@2
+            name: CalculateAndBumpModuleVersion
+            displayName: Calculate and bump module version
+            condition: succeeded()
+            inputs:
+              pwsh: true
+              targetType: inline
+              script: |
+                . "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug
+          - task: Bash@3
+            displayName: Push version bump changes
+            env:
+              GITHUB_TOKEN: $(GITHUB_TOKEN)
+            inputs:
+              targetType: inline
+              script: |
+                git status
+                git add "$(System.DefaultWorkingDirectory)/config/ModuleMetadata.json"
+                git commit -m 'Bump module versions after metadata generation.'
+                git push "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git"
+                git status
+
+        - ${{ if eq(parameters.CreatePullRequest, true) }}:
+          - template: .azure-pipelines/common-templates/create-pr.yml@self
+            parameters:
+              BaseBranch: $(BaseBranch)
+              TargetBranch: $(RefreshBranch)
+              Title: "[v2] Module Command Metadata Refresh"
+              Body: "This pull request was automatically created by Azure Pipelines. **Important** Check for unexpected deletions or changes in this PR."

.azure-pipelines/common-templates/download-openapi-docs.yml

@@ -2,17 +2,10 @@
 # Licensed under the MIT License.
 
 parameters:
-  - name: Branch
-    type: string
   - name: BaseBranch
     type: string
   - name: BuildAgent
     displayName: Build Agent
-  - name: SkipForceRefresh
-    type: string
-  - name: SkipOpenAPIDocsDownload
-    type: boolean
-    default: false
 
 steps:
   - template: ./checkout.yml
@@ -23,74 +16,49 @@ steps:
 
   - task: PowerShell@2
     name: "ComputeBranch"
-    displayName: "Compute weekly branch name"
+    displayName: "Compute weekly branch name and set Branch variable"
     inputs:
       targetType: inline
       script: |
-        $branch = "{0}/{1}" -f "$(Branch)", (Get-Date -Format yyyyMMddHHmm)
-        Write-Host "##vso[task.setvariable variable=WeeklyBranch;isOutput=true]$branch"
+        $branch = "WeeklyApiRefresh/{0}" -f (Get-Date -Format yyyyMMddHHmm)
+        Write-Host "Computed weekly branch: $branch"
+        # Set the Branch pipeline variable for current job
+        Write-Host "##vso[task.setvariable variable=Branch;isOutput=false]$branch"
+        # Set the Branch variable as output for subsequent jobs
+        Write-Host "##vso[task.setvariable variable=Branch;isOutput=true]$branch"
 
   - task: Bash@3
     displayName: "Create weekly branch"
     inputs:
       targetType: inline
       script: |
         git status
-        git branch $(ComputeBranch.WeeklyBranch)
-        git checkout $(ComputeBranch.WeeklyBranch)
+        git branch $(Branch)
+        git checkout $(Branch)
         git status
 
   - task: PowerShell@2
     displayName: Download v1.0 OpenApi docs
-    condition: and(succeeded(), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false))
     continueOnError: false
     inputs:
       filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1"
-      arguments: -SkipForceRefresh:$$(SkipForceRefresh)
       pwsh: true
 
   - task: PowerShell@2
     displayName: Download beta OpenApi docs
-    condition: and(succeeded(), eq('${{ parameters.SkipOpenAPIDocsDownload }}', false))
     continueOnError: false
     inputs:
       filePath: "$(System.DefaultWorkingDirectory)/tools/UpdateOpenApi.ps1"
-      arguments: -BetaGraphVersion -SkipForceRefresh:$$(SkipForceRefresh)
+      arguments: -BetaGraphVersion
       pwsh: true
 
   - task: PowerShell@2
     name: OpenAPIDocDiff
-    displayName: Get OpenAPI docs diff
+    displayName: Get OpenAPI docs diff and set ModuleGenerationList variable
     inputs:
+      filePath: "$(System.DefaultWorkingDirectory)/tools/Utilities/OpenApiDocDiff.ps1"
+      arguments: -OutputVariable "ModuleGenerationList"
       pwsh: true
-      targetType: "inline"
-      script: |
-        Write-Host "SkipOpenAPIDocsDownload: ${{ parameters.SkipOpenAPIDocsDownload }}"
-        if ('${{ parameters.SkipOpenAPIDocsDownload }}' -eq 'True') {
-          $ModuleNames = "Skipped"
-          Write-Warning "Skipped OpenAPI Docs Download."
-        } else  {
-          $diff = git diff --name-only
-          $ModulesWithChanges = @{}
-          $diff | %{
-            if (($_ -match 'openApiDocs\/(v1.0|beta)\/(.*).yml') -and !$ModulesWithChanges.ContainsKey($matches.2))
-            {
-              $ModulesWithChanges.Add($matches.2, $matches.1)
-            }
-          }
-          $ModuleNames = $ModulesWithChanges.Keys
-        }
-        Write-Host "##vso[task.setvariable variable=ModulesWithChanges;isOutput=true]$ModuleNames"
-
-  - task: PowerShell@2
-    name: CalculateAndBumpModuleVersion
-    displayName: Calculate and bump module version
-    condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], ''))
-    inputs:
-      pwsh: true
-      targetType: inline
-      script: |
-        . "$(System.DefaultWorkingDirectory)\tools\Versions\BumpModuleVersion.ps1" -BumpV1Module -BumpBetaModule -BumpAuthModule -Debug
 
   - task: PowerShell@2
     name: BuildOpenApiMetadataDetectionTool
@@ -112,7 +80,7 @@ steps:
 
   - task: Bash@3
     displayName: Commit downloaded files
-    condition: and(succeeded(), ne(variables['OpenAPIDocDiff.ModulesWithChanges'], ''))
+    condition: and(succeeded(), ne(variables['ModuleGenerationList'], ''))
     env:
       GITHUB_TOKEN: $(GITHUB_TOKEN)
     inputs:
@@ -122,10 +90,10 @@ steps:
         git add .
         git commit -m 'Weekly OpenApiDocs Download.'
         git status
-        git push --set-upstream "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git" $(ComputeBranch.WeeklyBranch)
+        git push --set-upstream "https://$(GITHUB_TOKEN)@github.com/microsoftgraph/msgraph-sdk-powershell.git" $(Branch)
         git status
 
 # References
 # [0] https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables
 # [1] https://hub.github.com/hub-pull-request.1.html
-# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token
+# https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token

.azure-pipelines/generation-templates/individual-workload-module.yml

@@ -0,0 +1,48 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+
+parameters:
+  - name: Test
+    type: boolean
+    default: true
+  - name: ModuleName
+    type: string
+    default: ""    
+  - name: ModuleVersion
+    type: string
+    default: ""
+  - name: Pack
+    type: boolean
+    default: true
+  - name: Sign
+    type: boolean
+    default: true
+
+steps:
+  - task: PowerShell@2
+    displayName: Generate Workload Modules
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -EnableSigning:$${{ parameters.Sign }} -Build -ExcludeExampleTemplates -ExcludeNotesSection -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
+  - template: ../common-templates/guardian-analyzer.yml
+
+  - task: PowerShell@2
+    displayName: Test Workload Modules
+    enabled: false
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/GenerateModules.ps1 -SkipGeneration -Test -ModuleToGenerate ${{ parameters.ModuleName }} -ApiVersion ${{ parameters.ModuleVersion }}
+
+  - task: PowerShell@2
+    displayName: Find Duplicate Commands
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        . $(System.DefaultWorkingDirectory)/tools/PostGeneration/FindDuplicateCommand.ps1 -SourcePath "$(System.DefaultWorkingDirectory)/src/"
+