Migrate pipeline linux test to docker image

.azure_pipelines/dockerfiles/linux-cpu.dockerfile

@@ -0,0 +1,21 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+FROM ubuntu:22.04
+
+ARG PYTHON_VERSION
+
+RUN apt-get update && \
+    apt-get install -y \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python${PYTHON_VERSION}-venv \
+    python3-pip \
+    unzip \
+    docker.io
+RUN ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python
+
+COPY . /olive
+WORKDIR /olive
+RUN pip install -e .

.azure_pipelines/dockerfiles/linux-gpu.dockerfile

@@ -0,0 +1,41 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+
+ARG PYTHON_VERSION
+ARG TENSORRT_VERSION=10.0.1.6-1+cuda12.4
+
+RUN apt-get update && \
+    apt-get install -y \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python${PYTHON_VERSION}-venv \
+    python3-pip \
+    libnvinfer10=${TENSORRT_VERSION} \
+    libnvinfer-dev=${TENSORRT_VERSION} \
+    libnvinfer-plugin-dev=${TENSORRT_VERSION} \
+    libnvinfer-vc-plugin-dev=${TENSORRT_VERSION} \
+    libnvinfer-headers-plugin-dev=${TENSORRT_VERSION} \
+    libnvonnxparsers-dev=${TENSORRT_VERSION} \
+    libnvinfer-plugin10=${TENSORRT_VERSION} \
+    libnvinfer-vc-plugin10=${TENSORRT_VERSION} \
+    libnvonnxparsers10=${TENSORRT_VERSION} \
+    libnvinfer-headers-dev=${TENSORRT_VERSION} \
+    libnvinfer-lean10=${TENSORRT_VERSION} \
+    python3-libnvinfer-lean=${TENSORRT_VERSION} \
+    libnvinfer-dispatch10=${TENSORRT_VERSION} \
+    python3-libnvinfer-dispatch=${TENSORRT_VERSION} \
+    tensorrt-libs=${TENSORRT_VERSION} \
+    tensorrt-dev=${TENSORRT_VERSION} \
+    libnvinfer-lean-dev=${TENSORRT_VERSION} \
+    libnvinfer-dispatch-dev=${TENSORRT_VERSION} \
+    python3-libnvinfer=${TENSORRT_VERSION} \
+    unzip \
+    docker.io
+RUN ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python
+
+COPY . /olive
+WORKDIR /olive
+RUN pip install -e .

.azure_pipelines/job_templates/build-docker-image-template.yaml

@@ -0,0 +1,19 @@
+# Docker image build template
+
+parameters:
+  dockerfile: ''
+  python_version: ''
+  docker_image: ''
+
+steps:
+- script: |
+    docker login -u $(docker-username) -p $(docker-password)
+    docker build --build-arg PYTHON_VERSION=${{ parameters.python_version }} -t ${{ parameters.docker_image }} -f $(Build.SourcesDirectory)/${{ parameters.dockerfile }} .
+  displayName: Build Docker Image
+
+- script: |
+    docker version
+    docker image ls
+    docker system df
+    df -h
+  displayName: Check Docker Images

.azure_pipelines/job_templates/olive-example-linux-template.yaml

@@ -0,0 +1,78 @@
+# Olive Build and Test Pipeline template for examples on Azure DevOps
+
+parameters:
+  name: ''
+  pool: ''
+  python_version: '3.10'
+  device: 'cpu'
+  dockerfile: '.azure_pipelines/dockerfiles/linux-cpu.dockerfile'
+  docker_image: 'olive-pipeline:latest'
+  onnxruntime: 'onnxruntime'
+  subfolder: 'local'
+  torch: 'torch'
+  test_script: 'run_test.sh'
+  onnxruntime_nightly: false
+
+jobs:
+  - job: ${{ parameters.name }}_Test_Examples
+    timeoutInMinutes: 300
+    pool:
+      name: ${{ parameters.pool }}
+    strategy:
+      matrix:
+        ${{ insert }}: ${{ parameters.examples }}
+    variables:
+      PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
+      HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
+      OLIVE_TEMPDIR: $(Pipeline.Workspace)/.olive_tempdir
+
+    steps:
+    - template: build-docker-image-template.yaml
+      parameters:
+        python_version: ${{ parameters.python_version }}
+        dockerfile: ${{ parameters.dockerfile }}
+        docker_image: ${{ parameters.docker_image }}
+
+    # set exampleRequirements to requirements.txt if user does not specify
+    - script:
+        echo "##vso[task.setvariable variable=exampleRequirements]requirements.txt"
+      displayName: Set exampleRequirements
+      condition: eq(variables['exampleRequirements'], '')
+
+    - script: |
+        GPU_OPTION=""
+        if [ "${{ parameters.device }}" = "gpu" ]; then
+            GPU_OPTION="--gpus=all"
+        fi
+        docker run \
+        $GPU_OPTION \
+        -v $(Build.SourcesDirectory)/logs:/logs \
+        -e WORKSPACE_SUBSCRIPTION_ID=$(workspace-subscription-id) \
+        -e WORKSPACE_RESOURCE_GROUP=$(workspace-resource-group) \
+        -e WORKSPACE_NAME=$(workspace-name) \
+        -e MANAGED_IDENTITY_CLIENT_ID=$(olive-1es-identity-client-id) \
+        -e PIPELINE_TEST_ACCOUNT_NAME=$(pipeline-test-account-name) \
+        -e PIPELINE_TEST_CONTAINER_NAME=$(pipeline-test-container-name) \
+        -e KEYVAULT_NAME=$(keyvault-name) \
+        -e HF_TOKEN=$(hf_token) \
+        ${{ parameters.docker_image }} \
+        bash .azure_pipelines/scripts/${{ parameters.test_script }} \
+        ${{ parameters.torch }} \
+        ${{ parameters.onnxruntime }} \
+        ${{ parameters.onnxruntime_nightly }} \
+        examples/$(exampleFolder)/$(exampleRequirements) \
+        examples/test/${{ parameters.subfolder }}/test_$(exampleName).py
+      displayName: Run Tests in Docker
+
+    # Step 3: Publish test results
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFiles: '**/logs/test_examples-TestOlive.xml'
+        testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
+        failTaskOnFailedTests: true
+      displayName: Publish Test Results
+
+    - script: sudo git clean -dfX
+      condition: always()
+      displayName: Clean remaining artifacts

.azure_pipelines/job_templates/olive-example-template.yaml → .azure_pipelines/job_templates/olive-example-win-template.yaml

@@ -3,7 +3,9 @@
 parameters:
   name: ''
   pool: ''
-  python_version: '3.8'
+  test_type: ''
+  device: 'cpu'
+  python_version: '3.10'
   onnxruntime: 'onnxruntime'
   subfolder: 'local'
   torch: 'torch'

.azure_pipelines/job_templates/olive-setup-template.yaml

@@ -1,5 +1,5 @@
 parameters:
-  python_version: '3.8'
+  python_version: '3.10'
   onnxruntime: 'onnxruntime'
   torch: torch
 

.azure_pipelines/job_templates/olive-test-template.yaml → .azure_pipelines/job_templates/olive-test-linux-cpu-template.yaml

@@ -4,7 +4,7 @@ parameters:
   test_type: ''
   windows: False
   device: 'cpu'
-  python_version: '3.8'
+  python_version: '3.10'
   onnxruntime: 'onnxruntime'
   torch: 'torch'
   requirements_file: 'requirements-test.txt'

.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml

@@ -0,0 +1,85 @@
+# Example Linux test template for Olive pipeline
+
+parameters:
+  name: ''
+  pool: ''
+  test_type: ''
+  device: 'cpu'
+  dockerfile: '.azure_pipelines/dockerfiles/linux-gpu.dockerfile'
+  docker_image: 'olive-pipeline:latest'
+  python_version: '3.10'
+  onnxruntime: 'onnxruntime'
+  torch: 'torch'
+  requirements_file: 'requirements-test.txt'
+  test_script: 'run_test.sh'
+  onnxruntime_nightly: false
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 300
+  pool:
+    name: ${{ parameters.pool}}
+  variables:
+    testType: ${{ parameters.test_type }}
+    python_version: ${{ parameters.python_version }}
+    requirements_file: ${{ parameters.requirements_file }}
+    PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
+    HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
+
+  steps:
+  - script: docker system df && docker system prune -a -f && docker system df
+    displayName: Clean docker images
+    continueOnError: true
+
+  - template: build-docker-image-template.yaml
+    parameters:
+      python_version: ${{ parameters.python_version }}
+      dockerfile: ${{ parameters.dockerfile }}
+      docker_image: ${{ parameters.docker_image }}
+
+  - script: |
+      docker run \
+      --gpus=all \
+      -v /var/run/docker.sock:/var/run/docker.sock \
+      -v $(Build.SourcesDirectory)/logs:/logs \
+      -e WORKSPACE_SUBSCRIPTION_ID=$(workspace-subscription-id) \
+      -e WORKSPACE_RESOURCE_GROUP=$(workspace-resource-group) \
+      -e WORKSPACE_NAME=$(workspace-name) \
+      -e MANAGED_IDENTITY_CLIENT_ID=$(olive-1es-identity-client-id) \
+      ${{ parameters.docker_image }} \
+      bash .azure_pipelines/scripts/${{ parameters.test_script }} \
+      ${{ parameters.torch }} \
+      ${{ parameters.onnxruntime }} \
+      ${{ parameters.onnxruntime_nightly }} \
+      test/$(requirements_file) \
+      test/$(testType)
+    displayName: Run Tests in Docker
+
+  - task: CredScan@3
+    displayName: 'Run CredScan'
+    inputs:
+      debugMode: false
+    continueOnError: true
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '**/*TestOlive*.xml'
+      testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
+      failTaskOnFailedTests: true
+    displayName: Upload pipeline run test results
+
+  # Code coverage requires
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk 7.0.x'
+    inputs:
+      version: 7.0.x
+
+  - task: PublishCodeCoverageResults@2
+    inputs:
+      summaryFileLocation: '**/coverage.xml'
+    displayName: Publish code coverage results
+
+  - script: sudo git clean -dfX
+    condition: always()
+    displayName: Clean remaining artifacts

.azure_pipelines/job_templates/olive-test-win-template.yaml

@@ -0,0 +1,95 @@
+# Example Windows test template for Olive pipeline
+
+parameters:
+  name: ''
+  pool: ''
+  test_type: ''
+  windows: False
+  device: 'cpu'
+  python_version: '3.10'
+  onnxruntime: 'onnxruntime'
+  torch: 'torch'
+  requirements_file: 'requirements-test.txt'
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 300
+  pool:
+    name: ${{ parameters.pool}}
+  variables:
+    WINDOWS: ${{ parameters.windows}}
+    testType: ${{ parameters.test_type }}
+    python_version: ${{ parameters.python_version }}
+    requirements_file: ${{ parameters.requirements_file }}
+    PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
+    HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
+    PYTEST_BASETEMP: $(Pipeline.Workspace)/.pytest_basetemp
+
+  steps:
+  - script: docker system df && docker system prune -a -f && docker system df
+    displayName: Clean docker images
+    continueOnError: true
+
+  - template: olive-setup-template.yaml
+    parameters:
+      python_version: ${{ parameters.python_version }}
+      onnxruntime: ${{ parameters.onnxruntime }}
+      torch: ${{ parameters.torch }}
+
+  - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}:
+    - script: |
+          call python -m pip install pytest
+          call curl --output openvino_toolkit.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0.1/windows/w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64.zip
+          call 7z x openvino_toolkit.zip
+          call w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64\\setupvars.bat
+          call python -m pip install numpy psutil coverage protobuf==3.20.3
+          call coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) --basetemp $(PYTEST_BASETEMP)
+          call coverage xml
+      displayName: Test Olive
+      env:
+        WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
+        WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
+        WORKSPACE_NAME: $(workspace-name)
+        MANAGED_IDENTITY_CLIENT_ID: $(olive-1es-identity-client-id)
+  - ${{ else }}:
+    - script: |
+          python -m pip install pytest
+          python -m pip install -r $(Build.SourcesDirectory)/test/$(requirements_file)
+
+          coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) --basetemp $(PYTEST_BASETEMP)
+          coverage xml
+      displayName: Test Olive
+      env:
+        WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
+        WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
+        WORKSPACE_NAME: $(workspace-name)
+        MANAGED_IDENTITY_CLIENT_ID: $(olive-1es-identity-client-id)
+
+  - task: CredScan@3
+    displayName: 'Run CredScan'
+    inputs:
+      debugMode: false
+    continueOnError: true
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '**/*TestOlive*.xml'
+      testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
+      failTaskOnFailedTests: true
+    displayName: Upload pipeline run test results
+
+  # Code coverage requires
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk 7.0.x'
+    inputs:
+      version: 7.0.x
+
+  - task: PublishCodeCoverageResults@2
+    inputs:
+      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
+    displayName: Publish code coverage results
+
+  - script: git clean -dfX
+    condition: always()
+    displayName: Clean remaining artifacts

.azure_pipelines/olive-aml-ci.yaml

@@ -15,7 +15,7 @@ pr: none
 
 jobs:
 # Linux examples test
-- template: job_templates/olive-example-template.yaml
+- template: job_templates/olive-example-linux-template.yaml
   parameters:
     name: Linux_CI
     pool: $(OLIVE_POOL_UBUNTU2004)
@@ -37,7 +37,7 @@ jobs:
         exampleRequirements: requirements-pipeline.txt
 
 # Windows examples test
-- template: job_templates/olive-example-template.yaml
+- template: job_templates/olive-example-win-template.yaml
   parameters:
     name: Windows_CI
     pool: $(OLIVE_POOL_WIN2019)