Merge pull request #215 from ProtonMail/feature/interop-test-suite

Add interoperability test suite to CI

.github/actions/build-gosop/action.yml

@@ -0,0 +1,55 @@
+name: 'build-gosop'
+description: 'Build gosop from the current branch'
+
+inputs:
+
+  gopenpgp-ref: 
+    description: 'gopenpgp branch tag or commit to build from'
+    required: true
+    default: ''
+
+  binary-location: 
+    description: 'Path for the gosop binary'
+    required: true
+    default: './gosop-${{ github.sha }}'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout gopenpgp
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ inputs.gopenpgp-ref }}
+        path: gopenpgp
+    # Build gosop
+    - name: Set up latest golang
+      uses: actions/setup-go@v3
+      with:
+        go-version: ^1.18
+    - name: Check out gosop
+      uses: actions/checkout@v3
+      with:
+        repository: ProtonMail/gosop
+        path: gosop
+    - name: Cache go modules
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+    - name: Build gosop
+      run: ./.github/test-suite/build_gosop.sh
+      shell: bash
+    # Test the binary
+    - name: Print gosop version
+      run: ./gosop/gosop version --extended
+      shell: bash
+    # Move and rename binary
+    - name: Move binary
+      run: mv gosop/gosop ${{ inputs.binary-location }}
+      shell: bash
+    
+    

.github/test-suite/build_gosop.sh

@@ -0,0 +1,4 @@
+cd gosop
+echo "replace github.com/ProtonMail/gopenpgp/v2 => ../gopenpgp" >> go.mod
+go get github.com/ProtonMail/gopenpgp/v2/crypto
+go build .

.github/test-suite/config.json.template

@@ -0,0 +1,28 @@
+{
+    "drivers": [
+      {
+        "id": "gosop-branch",
+        "path": "__GOSOP_BRANCH__"
+      },
+      {
+        "id": "gosop-main",
+        "path": "__GOSOP_MAIN__"
+      },
+      {
+        "path": "__SQOP__"
+      },
+      {
+        "path": "__GPGME_SOP__"
+      },
+      {
+        "path": "__SOP_OPENPGPJS__"
+      },
+      {
+        "path": "__RNP_SOP__"
+      }
+    ],
+    "rlimits": {
+      "DATA": 1073741824
+    }
+}
+  

.github/test-suite/prepare_config.sh

@@ -0,0 +1,12 @@
+CONFIG_TEMPLATE=$1
+CONFIG_OUTPUT=$2
+GOSOP_BRANCH=$3
+GOSOP_MAIN=$4
+cat $CONFIG_TEMPLATE \
+    | sed "s@__GOSOP_BRANCH__@${GOSOP_BRANCH}@g" \
+    | sed "s@__GOSOP_MAIN__@${GOSOP_MAIN}@g" \
+    | sed "s@__SQOP__@${SQOP}@g" \
+    | sed "s@__GPGME_SOP__@${GPGME_SOP}@g" \
+    | sed "s@__SOP_OPENPGPJS__@${SOP_OPENPGPJS}@g" \
+    | sed "s@__RNP_SOP__@${RNP_SOP}@g" \
+    > $CONFIG_OUTPUT

.github/workflows/sop-test-suite.yml

@@ -0,0 +1,125 @@
+name: SOP interoperability test suite
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  build-gosop:
+    name: Build gosop from branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build gosop from branch
+        uses: ./.github/actions/build-gosop
+        with: 
+          binary-location: ./gosop-${{ github.sha }}
+      # Upload as artifact
+      - name: Upload gosop artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: gosop-${{ github.sha }}
+          path: ./gosop-${{ github.sha }}
+
+  build-gosop-main:
+    name: Build gosop from main
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build gosop from branch
+        uses: ./.github/actions/build-gosop
+        with: 
+          gopenpgp-ref: main
+          binary-location: ./gosop-main
+      # Upload as artifact
+      - name: Upload gosop-main artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: gosop-main
+          path: ./gosop-main
+    
+
+  test-suite:
+    name: Run interoperability test suite
+    runs-on: ubuntu-latest
+    container: 
+      image: ghcr.io/protonmail/openpgp-interop-test-docker:pmfork
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.github_token }}
+    needs: 
+      - build-gosop
+      - build-gosop-main
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      # Fetch gosop from main
+      - name: Download gosop-main
+        uses: actions/download-artifact@v3
+        with:
+          name: gosop-main
+      # Test gosop-main
+      - name: Make gosop-main executable
+        run: chmod +x gosop-main
+      - name: Print gosop-main version
+        run: ./gosop-main version --extended
+      # Fetch gosop from branch
+      - name: Download gosop-branch
+        uses: actions/download-artifact@v3
+        with:
+          name: gosop-${{ github.sha }}
+      - name: Rename gosop-branch
+        run: mv gosop-${{ github.sha }} gosop-branch
+      # Test gosop-branch
+      - name: Make gosop-branch executable
+        run: chmod +x gosop-branch
+      - name: Print gosop-branch version
+        run: ./gosop-branch version --extended
+      # Run test suite
+      - name: Prepare test configuration
+        run: ./.github/test-suite/prepare_config.sh $CONFIG_TEMPLATE $CONFIG_OUTPUT $GITHUB_WORKSPACE/gosop-branch $GITHUB_WORKSPACE/gosop-main
+        env:
+         CONFIG_TEMPLATE: .github/test-suite/config.json.template
+         CONFIG_OUTPUT: .github/test-suite/config.json
+      - name: Display configuration
+        run: cat .github/test-suite/config.json
+      - name: Run interoperability test suite
+        run: cd $TEST_SUITE_DIR && $TEST_SUITE --config $GITHUB_WORKSPACE/$CONFIG --json-out $GITHUB_WORKSPACE/$RESULTS_JSON --html-out $GITHUB_WORKSPACE/$RESULTS_HTML
+        env:
+         CONFIG: .github/test-suite/config.json
+         RESULTS_JSON: .github/test-suite/test-suite-results.json
+         RESULTS_HTML: .github/test-suite/test-suite-results.html
+      # Upload results
+      - name: Upload test results json artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-suite-results.json
+          path: .github/test-suite/test-suite-results.json
+      - name: Upload test results html artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-suite-results.html
+          path: .github/test-suite/test-suite-results.html
+
+  compare-with-main:
+    name: Compare with main
+    runs-on: ubuntu-latest
+    needs: test-suite
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Download test results json artifact
+        id: download-test-results
+        uses: actions/download-artifact@v3
+        with:
+          name: test-suite-results.json
+      - name: Compare with baseline
+        uses: ProtonMail/openpgp-interop-test-analyzer@v1
+        with: 
+          results: ${{ steps.download-test-results.outputs.download-path }}/test-suite-results.json
+          output: baseline-comparison.json
+          baseline: gosop-main
+          target: gosop-branch