ci, nix: build and cache all packages and CI dependencies

from f5bd9bc45e with changes for GitHub CI and misc Co-authored-by: Charles Hall <charles@computer.surgery> Signed-off-by: strawberry <strawberry@puppygock.gay>
2024-05-01 23:30:49 -04:00 · 2024-05-01 23:30:49 -04:00 · cad16b9268
parent 3b410d0556
commit cad16b9268
5 changed files with 93 additions and 55 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -53,7 +53,7 @@ jobs:

            - name: Enable Cachix binary cache
              run: |
-                  nix-env -iA cachix -f https://cachix.org/api/v1/install
+                  nix profile install nixpkgs#cachix
                  cachix use crane
                  cachix use nix-community

@ -78,7 +78,7 @@ jobs:
            - name: Prepare build environment
              run: |
                  echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix-env -f "<nixpkgs>" -iA direnv -iA nix-direnv
+                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
                  direnv allow
                  nix develop --command true

@ -144,9 +144,9 @@ jobs:
            - name: Install Nix
              uses: DeterminateSystems/nix-installer-action@main

-            - name: Enable Cachix binary cache
+            - name: Install and enable Cachix binary cache
              run: |
-                  nix-env -iA cachix -f https://cachix.org/api/v1/install
+                  nix profile install nixpkgs#cachix
                  cachix use crane
                  cachix use nix-community

@ -171,13 +171,13 @@ jobs:
            - name: Prepare build environment
              run: |
                  echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix-env -f "<nixpkgs>" -iA direnv -iA nix-direnv
+                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
                  direnv allow
                  nix develop --command true

            - name: Build static ${{ matrix.target }}
              run: |
-                  bin/nix-build-and-cache .#static-${{ matrix.target }}
+                  bin/nix-build-and-cache just .#static-${{ matrix.target }}
                  mkdir -p target/release
                  cp -v -f result/bin/conduit target/release/
                  direnv exec . cargo deb --no-build --no-strip --output target/debian/${{ matrix.target }}.deb
@ -199,7 +199,7 @@ jobs:

            - name: Build OCI image ${{ matrix.target }}
              run: |
-                  bin/nix-build-and-cache .#oci-image-${{ matrix.target }}
+                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}
                  cp -v -f result oci-image-${{ matrix.target }}.tar.gz

            - name: Upload OCI image ${{ matrix.target }}
--- a/.github/workflows/documentation.yml
+++ b/.github/workflows/documentation.yml
@ -88,13 +88,13 @@ jobs:
      - name: Allow direnv
        run: direnv allow

-      - name: Cache x86_64 inputs for devShell
+      - name: Cache CI dependencies
        run: |
-          ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+          ./bin/nix-build-and-cache ci

      - name: Build documentation (book)
        run: |
-          ./bin/nix-build-and-cache .#book
+          ./bin/nix-build-and-cache just .#book
          cp -r --dereference result public
      - name: Upload generated documentation (book) as normal artifact
        uses: actions/upload-artifact@v4
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -56,8 +56,8 @@ ci:
  stage: ci
  image: nixos/nix:2.22.0
  script:
-    # Cache the inputs required for the devShell
-    - ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation
+    # Cache CI dependencies
+    - ./bin/nix-build-and-cache ci

    - direnv exec . engage
  cache:
@ -81,7 +81,7 @@ artifacts:
  stage: artifacts
  image: nixos/nix:2.22.0
  script:
-    - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl
    - cp result/bin/conduit x86_64-unknown-linux-musl

    - mkdir -p target/release
@ -97,16 +97,16 @@ artifacts:
    # Note that although we have an `oci-image-x86_64-unknown-linux-musl`
    # output, we don't build it because it would be largely redundant to this
    # one since it's all containerized anyway.
-    - ./bin/nix-build-and-cache .#oci-image
+    - ./bin/nix-build-and-cache just .#oci-image
    - cp result oci-image-amd64.tar.gz

-    - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-aarch64-unknown-linux-musl
    - cp result/bin/conduit aarch64-unknown-linux-musl

-    - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#oci-image-aarch64-unknown-linux-musl
    - cp result oci-image-arm64v8.tar.gz

-    - ./bin/nix-build-and-cache .#book
+    - ./bin/nix-build-and-cache just .#book
    # We can't just copy the symlink, we need to dereference it https://gitlab.com/gitlab-org/gitlab/-/issues/19746
    - cp -r --dereference result public
  artifacts:
--- a/bin/complement
+++ b/bin/complement
@ -17,7 +17,9 @@ RESULTS_FILE="$3"

 OCI_IMAGE="complement-conduit:dev"

-pushd "$(git rev-parse --show-toplevel)" > /dev/null
+toplevel="$(git rev-parse --show-toplevel)"
+
+pushd "$toplevel" > /dev/null
 # uses nix-output-monitor (nom) if available
 if command -v nom &> /dev/null; then
    nom build .#complement
--- a/bin/nix-build-and-cache
+++ b/bin/nix-build-and-cache
@ -2,45 +2,81 @@

 set -eo pipefail

-# The first argument must be the desired installable
-INSTALLABLE="$1"
+toplevel="$(git rev-parse --show-toplevel)"

-# Build the installable and forward any other arguments too
-# uses nix-output-monitor (nom) if available
-if command -v nom &> /dev/null; then
-    nom build "$@"
-else
-    nix build -L "$@"
-fi
+# Build just the single installable and forward any other arguments too
+just() {
+    # uses nix-output-monitor (nom) if available
+    if command -v nom &> /dev/null; then
+        nom build "$@"
+    else
+        nix build -L "$@"
+    fi

-if [ ! -z "$ATTIC_TOKEN" ]; then
-    nix run --inputs-from . attic -- \
-        login \
-        conduit \
-        "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \
-        "$ATTIC_TOKEN"
+    if [ ! -z "$ATTIC_TOKEN" ]; then
+        # historical "conduit" store for compatibility purposes, same as conduwuit
+        nix run --inputs-from "$toplevel" attic -- \
+            login \
+            conduit \
+            "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduit}" \
+            "$ATTIC_TOKEN"

-    # Push the target installable and its build dependencies
-    nix run --inputs-from . attic -- \
-        push \
-        conduit \
-        "$(nix path-info "$INSTALLABLE" --derivation)" \
-        "$(nix path-info "$INSTALLABLE")"
+        readarray -t outputs < <(nix path-info "$@")
+        readarray -t derivations < <(nix path-info "$@" --derivation)
+
+        # Push the target installable and its build dependencies
+        nix run --inputs-from "$toplevel" attic -- \
+            push \
+            conduit \
+            "${outputs[@]}" \
+            "${derivations[@]}"
+
+        # main "conduwuit" store
+        nix run --inputs-from "$toplevel" attic -- \
+            login \
+            conduwuit \
+            "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \
+            "$ATTIC_TOKEN"
+
+        # Push the target installable and its build dependencies
+        nix run --inputs-from "$toplevel" attic -- \
+            push \
+            conduwuit \
+            "${outputs[@]}" \
+            "${derivations[@]}"
+    else
+        echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
+    fi
+
+}
+
+# Build and cache things needed for CI
+ci() {
+    cache=(
+        --inputs-from "$toplevel"
+
+        # Keep sorted
+        "$toplevel#devShells.x86_64-linux.default.inputDerivation"
+        attic#default
+        nixpkgs#direnv
+        nixpkgs#jq
+        nixpkgs#nix-direnv
+    )
+
+    just "${cache[@]}"
+}
+
+# Build and cache *all* the package outputs from the flake.nix
+packages() {
+    declare -a cache="($(
+        nix flake show --json 2> /dev/null |
+            nix run --inputs-from "$toplevel" nixpkgs#jq -- \
+            -r \
+            '.packages."x86_64-linux" | keys | map("'"$toplevel"'#" + .) | @sh'
+    ))"
+
+    just "${cache[@]}"
+}


-    # push to "conduwuit" too
-    nix run --inputs-from . attic -- \
-        login \
-        conduwuit \
-        "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \
-        "$ATTIC_TOKEN"
-
-    # Push the target installable and its build dependencies
-    nix run --inputs-from . attic -- \
-        push \
-        conduwuit \
-        "$(nix path-info "$INSTALLABLE" --derivation)" \
-        "$(nix path-info "$INSTALLABLE")"
-else
-    echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
-fi
+eval "$@"