chore: improve CI workflow speed and usage

7 months ago · f2b12469f3
parent 0ef9a0e2c6
commit f2b12469f3
9 changed files with 173 additions and 172 deletions
--- a/.github/workflows/build-binaries.yml
+++ b/.github/workflows/build-binaries.yml
@ -1,3 +1,7 @@
+# There is a fair bit of duplication here, but it is the best to save our github free minutes for now.
+# We could save and restore cache to different jobs but that takes roughly 3 minutes to save,
+# so better run them in parrallel instead.
+
 name: Session Desktop

 on:
@ -18,55 +22,141 @@ concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

+env:
+  # we want to publish on "push to master" only. When we don't want to publish, we want to upload artefacts
+  SHOULD_PUBLISH: ${{ github.event_name == 'push' && github.ref == 'master' }}
+
 jobs:
-  build:
-    runs-on: ${{ matrix.os }}
+  build_linux:
+    runs-on: ubuntu-20.04
    strategy:
-      fail-fast: false
      matrix:
-        # We want a mac arm64 build, and according to this https://github.com/actions/runner-images#available-images macos-14 is always arm64
-        # macos-14 is disabled for now as we hit our free tier limit for macos builds
-        os: [windows-2022, ubuntu-20.04, macos-12]
+        # this needs to be a valid target of https://www.electron.build/linux#target
+        pkg_to_build: ['deb', 'rpm', 'freebsd', 'AppImage']
+
    env:
-      SIGNAL_ENV: production
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - run: git config --global core.autocrlf false

      - name: Checkout git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Setup & Build
        uses: ./actions/setup_and_build
+        with:
+          cache_suffix: ${{ matrix.pkg_to_build }}

      - name: Lint Files
-        # no need to lint files on all platforms. Just do it once on the quicker one
-        if: runner.os == 'Linux'
+        # no need to lint files on all platforms
        run: yarn lint-full

      - name: Enforce yarn.lock has no duplicates
+        # no need to dedup yarn.lock on all platforms
        uses: ./actions/deduplicate_fail
+
+        # we want to test on all platforms are some are testing the menus rendered (and are depent on the platform)
+      - name: Unit Test
+        run: yarn test
+
+      - name: Build but do not publish ${{ matrix.pkg_to_build }}
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        run: |
+          sed -i 's/"target": \["deb"\]/"target": "${{ matrix.pkg_to_build }}"/g' package.json && yarn build-release
+
+      - name: Upload artefacts ${{ matrix.pkg_to_build }}
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        uses: ./actions/upload_prod_artefacts
+        with:
+          upload_prefix: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.pkg_to_build }}
+
+      - name: Build & publish ${{ matrix.pkg_to_build }}
+        # we want this to run only when on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'true' }}
+        run: |
+          sed -i 's/\"target\": \\[\"deb\"\\]/\"target\": \"${{ matrix.pkg_to_build }}\"/g' package.json; yarn build-release-publish
+
+  build_windows:
+    runs-on: windows-2022
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - run: git config --global core.autocrlf false
+
+      - name: Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: Setup & Build
+        uses: ./actions/setup_and_build
+        with:
+          cache_suffix: 'windows_x64'
+
+        # we want to test on all platforms are some are testing the menus rendered (and are depent on the platform)
      - name: Unit Test
        run: yarn test

      - name: Build but do not publish
        # we want this to run always, except on "push" to "master"
-        if: github.event_name != 'push' || github.ref != 'master'
-        uses: ./actions/build_no_publish
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        run: yarn build-release
+
+      - name: Upload artefacts
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        uses: ./actions/upload_prod_artefacts
        with:
-          MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }}
-          MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }}
-          SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }}
-          SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }}
-          SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }}
+          upload_prefix: ${{ runner.os }}-${{ runner.arch }}

      - name: Build & publish
        # we want this to run only when on "push" to "master"
-        if: github.event_name == 'push' && github.ref == 'master'
-        uses: ./actions/build_publish
+        if: ${{ env.SHOULD_PUBLISH == 'true' }}
+        run: yarn build-release-publish # No other args needed for windows publish
+
+  # We want a mac arm64 build, and according to this https://github.com/actions/runner-images#available-images macos-14 is always arm64
+  # macos-14 is disabled for now as we hit our free tier limit for macos builds
+  build_macos_x64:
+    runs-on: macos-12
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }}
+      MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }}
+      SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }}
+      SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }}
+      SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }}
+    steps:
+      - run: git config --global core.autocrlf false
+
+      - name: Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: Setup & Build
+        uses: ./actions/setup_and_build
        with:
-          MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }}
-          MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }}
-          SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }}
-          SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }}
-          SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }}
+          cache_suffix: 'macos_x64'
+
+        # we want to test on all platforms are some are testing the menus rendered (and are depent on the platform)
+      - name: Unit Test
+        run: yarn test
+
+      - name: Build but do not publish
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        run: |
+          source ./build/setup-mac-certificate.sh
+          yarn build-release --config.mac.bundleVersion=${{ github.ref }}
+
+      - name: Upload artefacts
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        uses: ./actions/upload_prod_artefacts
+        with:
+          upload_prefix: ${{ runner.os }}-${{ runner.arch }}
+
+      - name: Build & publish
+        # we want this to run only when on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'true' }}
+        run: |
+          source ./build/setup-mac-certificate.sh
+          yarn build-release-publish --config.mac.bundleVersion=${{ github.ref }}
--- a/actions/build_no_publish/action.yml
+++ b/actions/build_no_publish/action.yml
@ -1,64 +0,0 @@
-name: 'Build Only (no publish)'
-description: 'Build only (no publish)'
-inputs:
-  MAC_CERTIFICATE:
-    description: 'MAC_CERTIFICATE (mac build only)'
-    required: true
-  MAC_CERTIFICATE_PASSWORD:
-    description: 'MAC_CERTIFICATE_PASSWORD (mac build only)'
-    required: true
-  SIGNING_APPLE_ID:
-    description: 'SIGNING_APPLE_ID (mac build only)'
-    required: true
-  SIGNING_APP_PASSWORD:
-    description: 'SIGNING_APP_PASSWORD (mac build only)'
-    required: true
-  SIGNING_TEAM_ID:
-    description: 'SIGNING_TEAM_ID (mac build only)'
-    required: true
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Build windows production binaries
-      shell: bash
-      if: runner.os == 'Windows'
-      run: $(yarn bin)/electron-builder --config.extraMetadata.environment=%SIGNAL_ENV% --publish=never --config.directories.output=release
-
-    - name: Build mac production binaries
-      shell: bash
-      if: runner.os == 'macOS'
-      run: |
-        source ./build/setup-mac-certificate.sh
-        $(yarn bin)/electron-builder --config.extraMetadata.environment=$SIGNAL_ENV --config.mac.bundleVersion=${{ github.ref }} --publish=never --config.directories.output=release
-      env:
-        MAC_CERTIFICATE: ${{ inputs.MAC_CERTIFICATE }}
-        MAC_CERTIFICATE_PASSWORD: ${{ inputs.MAC_CERTIFICATE_PASSWORD }}
-        SIGNING_APPLE_ID: ${{ inputs.SIGNING_APPLE_ID }}
-        SIGNING_APP_PASSWORD: ${{ inputs.SIGNING_APP_PASSWORD }}
-        SIGNING_TEAM_ID: ${{ inputs.SIGNING_TEAM_ID }}
-
-    - name: Build linux production binaries
-      shell: bash
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt-get install -y rpm
-        yarn build-release
-
-    - name: Remove unpacked files
-      run: |
-        ls -d -- */ | xargs -I{} echo "Removing {}"
-        ls -d -- */ | xargs -I{} rm -rf {}
-      shell: bash
-      working-directory: ./release/
-
-    - name: Remaining files
-      run: ls .
-      shell: bash
-      working-directory: ./release/
-
-    - name: Upload Production Artifacts
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ runner.os }}-${{ runner.arch }}-production
-        path: release
--- a/actions/build_publish/action.yml
+++ b/actions/build_publish/action.yml
@ -1,29 +0,0 @@
-name: 'Build & Publish'
-description: 'Build & Publish'
-runs:
-  using: 'composite'
-  steps:
-    - name: Build & publish windows production binaries
-      shell: bash
-      if: runner.os == 'Windows' && github.ref == 'master' && github.event_name == 'push'
-      run: $(yarn bin)/electron-builder --config.extraMetadata.environment=%SIGNAL_ENV% --publish=always
-
-    - name: Build & publish mac production binaries
-      shell: bash
-      if: runner.os == 'macOS' && github.ref == 'master' && github.event_name == 'push'
-      run: |
-        source ./build/setup-mac-certificate.sh
-        $(yarn bin)/electron-builder --config.extraMetadata.environment=$SIGNAL_ENV --config.mac.bundleVersion=${{ github.ref }} --publish=always
-      env:
-        MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }}
-        MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }}
-        SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }}
-        SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }}
-        SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }}
-
-    - name: Build & publish linux production binaries
-      shell: bash
-      if: runner.os == 'Linux' && github.ref == 'master' && github.event_name == 'push'
-      run: |
-        sudo apt-get install -y rpm
-        yarn build-release-publish
--- a/actions/setup_and_build/action.yml
+++ b/actions/setup_and_build/action.yml
@ -1,29 +1,25 @@
-
 name: 'Setup and build'
 description: 'Setup and build Session Desktop'
+inputs:
+  cache_suffix:
+    description: 'the package we are currently building (used as key for the cached node_modules)'
+    required: true
+
 runs:
  using: 'composite'
  steps:
    - name: Install node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
      with:
        node-version-file: '.nvmrc'

-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'

-    - name: Cache Desktop node_modules
-      id: cache-desktop-modules
-      uses: actions/cache@v3
-      if: runner.os != 'Windows'
-      with:
-        path: node_modules
-        key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }}
-
    # Not having this will break the windows build because the PATH won't be set by msbuild.
    - name: Add msbuild to PATH
-      uses: microsoft/setup-msbuild@v1.3.1
+      uses: microsoft/setup-msbuild@v2
      if: runner.os == 'Windows'

    - name: Setup node for windows
@ -32,11 +28,24 @@ runs:
      run: |
        yarn global add node-gyp@latest

+    - uses: actions/cache/restore@v4
+      id: cache-desktop-modules
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache_suffix }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }}
+
    - name: Install dependencies
      shell: bash
      if: steps.cache-desktop-modules.outputs.cache-hit != 'true'
      run: yarn install --frozen-lockfile --network-timeout 600000

+    - uses: actions/cache/save@v4
+      id: cache-desktop-modules-save
+      if: runner.os != 'Windows'
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache_suffix }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }}
+
    - name: Generate and concat files
      shell: bash
      run: yarn build-everything
--- a/actions/upload_prod_artefacts/action.yml
+++ b/actions/upload_prod_artefacts/action.yml
@ -0,0 +1,27 @@
+name: 'Upload production artefact (not publish)'
+description: 'Upload production artefact (not publish)'
+inputs:
+  upload_prefix:
+    description: 'upload name prefix'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Remove unpacked files
+      run: |
+        ls -d -- */ | xargs -I{} echo "Removing {}"
+        ls -d -- */ | xargs -I{} rm -rf {}
+      shell: bash
+      working-directory: ./release/
+
+    - name: Remaining files
+      run: ls .
+      shell: bash
+      working-directory: ./release/
+
+    - name: Upload Production Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.upload_prefix }}-production
+        path: release
--- a/build/updateLocalConfig.js
+++ b/build/updateLocalConfig.js
@ -3,7 +3,7 @@ var _ = require('lodash');
 var execSync = require('child_process').execSync;

 const updateLocalConfig = () => {
-  var environment = process.env.SIGNAL_ENV || 'production';
+  var environment = 'production';
  var configPath = `config/local-${environment}.json`;
  var localConfig;

--- a/package.json
+++ b/package.json
@ -49,19 +49,9 @@
    "format-full": "prettier --list-different --write \"*.{css,js,json,scss,ts,tsx}\" \"./**/*.{css,js,json,scss,ts,tsx}\"",
    "start-prod-test": "cross-env NODE_ENV=production NODE_APP_INSTANCE=$MULTI electron .",
    "test": "mocha",
-    "build-release": "run-script-os",
-    "build-release-non-linux": "yarn build-everything && cross-env SIGNAL_ENV=production electron-builder --config.extraMetadata.environment=production --publish=never --config.directories.output=release",
-    "build-release:win32": "yarn build-release-non-linux",
-    "build-release:macos": "yarn build-release-non-linux",
-    "build-release:linux": "yarn sedtoDeb; yarn build-release-non-linux && yarn sedtoAppImage && yarn build-release-non-linux && yarn sedtoDeb",
-    "build-release-publish": "run-script-os",
-    "build-release-publish-non-linux": "yarn build-everything && cross-env SIGNAL_ENV=production $(yarn bin)/electron-builder --config.extraMetadata.environment=$SIGNAL_ENV --publish=always",
-    "build-release-publish:win32": "yarn build-release-publish-non-linux",
-    "build-release-publish:macos": "yarn build-release-publish-non-linux",
-    "build-release-publish:linux": "yarn sedtoDeb; yarn build-release-publish-non-linux && yarn sedtoAppImage && yarn build-release-publish-non-linux && yarn sedtoDeb",
-    "appImage": "yarn sedtoAppImage; yarn build-release-non-linux; yarn sedtoDeb",
-    "sedtoAppImage": "sed -i 's/\"target\": \\[\"deb\", \"rpm\", \"freebsd\"\\]/\"target\": \"AppImage\"/g' package.json",
-    "sedtoDeb": "sed -i 's/\"target\": \"AppImage\"/\"target\": \\[\"deb\", \"rpm\", \"freebsd\"\\]/g' package.json",
+    "build-release-base": "cross-env NODE_ENV=production electron-builder --config.extraMetadata.environment=production",
+    "build-release": "yarn build-release-base --publish=never --config.directories.output=release",
+    "build-release-publish": "yarn build-release-base --publish=always",
    "ready": "yarn dedup-yarn-lock --fail && yarn build-everything && yarn lint-full && yarn test",
    "postinstall": "yarn patch-package && yarn electron-builder install-app-deps",
    "update-git-info": "node ./build/updateLocalConfig.js",
@ -207,7 +197,6 @@
    "postinstall-prepare": "^1.0.1",
    "prettier": "3.2.5",
    "protobufjs-cli": "^1.1.1",
-    "run-script-os": "^1.1.6",
    "sass": "^1.60.0",
    "sass-loader": "^13.2.2",
    "sinon": "9.0.2",
@ -272,7 +261,7 @@
        "StartupWMClass": "Session"
      },
      "asarUnpack": "node_modules/spellchecker/vendor/hunspell_dictionaries",
-      "target": ["deb", "rpm", "freebsd"],
+      "target": ["deb"],
      "icon": "build/icon-linux.icns"
    },
    "asarUnpack": [
@ -294,8 +283,8 @@
    "files": [
      "package.json",
      "config/default.json",
-      "config/${env.SIGNAL_ENV}.json",
-      "config/local-${env.SIGNAL_ENV}.json",
+      "config/production.json",
+      "config/local-production.json",
      "background.html",
      "about.html",
      "password.html",
--- a/ts/test/session/unit/updater/updater_test.ts
+++ b/ts/test/session/unit/updater/updater_test.ts
@ -6,14 +6,15 @@ import { isEmpty } from 'lodash';
 describe('Updater', () => {
  it.skip('isUpdateAvailable', () => {});

-  it('package.json target are correct', () => {
+  it('package.json target is correct', () => {
    const content = readFileSync(
      path.join(__dirname, '..', '..', '..', '..', '..', 'package.json')
    );

-    if (!content || isEmpty(content) || !content.includes('"target": ["deb", "rpm", "freebsd"],')) {
+    // the CI for building release relies on this being set to build the different targets.
+    if (!content || isEmpty(content) || !content.includes('"target": ["deb"],')) {
      throw new Error(
-        'Content empty or does not contain the target on a single line. They have to be for the linux appImage build to pass.'
+        'Content empty or does not contain the target on a single line. They have to be for the linux CI builds to pass.'
      );
    }
  });
--- a/yarn.lock
+++ b/yarn.lock
@ -765,13 +765,6 @@
  resolved "https://registry.yarnpkg.com/@types/filesize/-/filesize-3.6.0.tgz#5f1a25c7b4e3d5ee2bc63133d374d096b7008c8d"
  integrity sha512-rOWxCKMjt2DBuwddUnl5GOpf/jAkkqteB+XldncpVxVX+HPTmK2c5ACMOVEbp9gaH81IlhTdC3TwvRa5nopasw==

-"@types/firstline@^2.0.2":
-  version "2.0.4"
-  resolved "https://registry.yarnpkg.com/@types/firstline/-/firstline-2.0.4.tgz#b8d3f8f7396d1589efea89db183c047a42efaf04"
-  integrity sha512-EYoMzk783ncj3soLGADXD/rklDMw1PAO5Hc3lRZa5G21vkfacwkdTlIdhTJ39omqDLezTSmxjDG1psd4A/mUHg==
-  dependencies:
-    "@types/node" "*"
-
 "@types/fs-extra@5.0.5":
  version "5.0.5"
  resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-5.0.5.tgz#080d90a792f3fa2c5559eb44bd8ef840aae9104b"
@ -3674,11 +3667,6 @@ find-yarn-workspace-root@^2.0.0:
  dependencies:
    micromatch "^4.0.2"

-firstline@1.2.1:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/firstline/-/firstline-1.2.1.tgz#b88673c42009f8821fac2926e99720acee924fae"
-  integrity sha512-6eMQNJtDzyXSC1yeCBWspqA6LeV5la2XHGTXQq4O0xkglAutpyny/sB+zVdXTZ9nzcDW9ZGLxwXXkB+ZEtJuPw==
-
 flat-cache@^3.0.4:
  version "3.2.0"
  resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.2.0.tgz#2c0c2d5040c99b1632771a9d105725c0115363ee"
@ -6381,11 +6369,6 @@ read-config-file@6.2.0:
    json5 "^2.2.0"
    lazy-val "^1.0.4"

-read-last-lines-ts@^1.2.1:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/read-last-lines-ts/-/read-last-lines-ts-1.2.1.tgz#99e46288c5373c06e16e90e666a46b595dad80a1"
-  integrity sha512-1VcCrAU38DILYiF4sbNY13zdrMGwrFqjGQnXJy28G1zLJItvnWtgCbqoAJlnZZSiEICMKdM4Ol7LYvVMEoKrAg==
-
 read-pkg-up@^7.0.1:
  version "7.0.1"
  resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz#f3a6135758459733ae2b95638056e1854e7ef507"
@ -6695,11 +6678,6 @@ run-parallel@^1.1.9:
  dependencies:
    queue-microtask "^1.2.2"

-run-script-os@^1.1.6:
-  version "1.1.6"
-  resolved "https://registry.yarnpkg.com/run-script-os/-/run-script-os-1.1.6.tgz#8b0177fb1b54c99a670f95c7fdc54f18b9c72347"
-  integrity sha512-ql6P2LzhBTTDfzKts+Qo4H94VUKpxKDFz6QxxwaUZN0mwvi7L3lpOI7BqPCq7lgDh3XLl0dpeXwfcVIitlrYrw==
-
 safe-array-concat@^1.1.2:
  version "1.1.2"
  resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.2.tgz#81d77ee0c4e8b863635227c721278dd524c20edb"