From 5a26f392ca640818da0b8590be6119699e852b07 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 19 Jan 2024 10:12:31 -0800 Subject: [PATCH] Use the correct Docker Image for RocksJava on Linux (#12169) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/12169 Reviewed By: pdillinger Differential Revision: D52715225 Pulled By: ajkr fbshipit-source-id: 28476d363034fa1bb9c8c919d577c03b6391451b --- .circleci/config.yml | 21 +++++------ .github/actions/install-maven/action.yml | 8 +++-- .github/actions/post-benchmarks/action.yml | 7 ++-- .github/actions/post-pmd-steps/action.yml | 10 ------ .github/actions/post-steps/action.yml | 41 ++++++++++++++-------- .github/workflows/pr-jobs.yml | 38 +++++++++++++++----- 6 files changed, 78 insertions(+), 47 deletions(-) delete mode 100644 .github/actions/post-pmd-steps/action.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index cebae35ec7..d47c18c7c1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,9 +146,12 @@ commands: install-maven: steps: - run: - name: Install maven + name: Install Maven command: | - sudo apt-get update -y && sudo apt-get install -y maven + wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz + tar zxf apache-maven-3.9.6-bin.tar.gz + echo "export M2_HOME=$(pwd)/apache-maven-3.9.6" >> $BASH_ENV + echo 'export PATH=$M2_HOME/bin:$PATH' >> $BASH_ENV setup-folly: steps: @@ -231,6 +234,7 @@ executors: - image: zjay437/rocksdb:0.6 linux-java-docker: docker: + # This is the Docker Image used for building RocksJava releases, see: https://github.com/evolvedbinary/docker-rocksjava - image: evolvedbinary/rocksjava:centos6_x64-be jobs: @@ -623,7 +627,7 @@ jobs: - windows-build-steps build-linux-java: - executor: linux-docker + executor: linux-java-docker resource_class: large steps: - pre-steps @@ -636,17 +640,13 @@ jobs: which javac && javac -version - run: name: "Test RocksDBJava" - command: make V=1 J=8 -j8 jtest + command: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest' - post-steps build-linux-java-pmd: - machine: - image: ubuntu-2004:202111-02 + executor: linux-java-docker resource_class: large - environment: - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 steps: - - install-maven - pre-steps - run: name: "Set Java Environment" @@ -655,9 +655,10 @@ jobs: echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV which java && java -version which javac && javac -version + - install-maven - run: name: "PMD RocksDBJava" - command: make V=1 J=8 -j8 jpmd + command: scl enable devtoolset-7 'make V=1 J=8 -j8 jpmd' - post-pmd-steps build-linux-java-static: diff --git a/.github/actions/install-maven/action.yml b/.github/actions/install-maven/action.yml index be54927f77..69a925272a 100644 --- a/.github/actions/install-maven/action.yml +++ b/.github/actions/install-maven/action.yml @@ -2,6 +2,10 @@ name: install-maven runs: using: composite steps: - - name: Install maven - run: sudo apt-get update -y && sudo apt-get install -y maven + - name: Install Maven + run: | + wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz + tar zxf apache-maven-3.9.6-bin.tar.gz + echo "export M2_HOME=$(pwd)/apache-maven-3.9.6" >> $GITHUB_ENV + echo "$(pwd)/apache-maven-3.9.6/bin" >> $GITHUB_PATH shell: bash diff --git a/.github/actions/post-benchmarks/action.yml b/.github/actions/post-benchmarks/action.yml index e2466f4606..46d47ae223 100644 --- a/.github/actions/post-benchmarks/action.yml +++ b/.github/actions/post-benchmarks/action.yml @@ -2,9 +2,12 @@ name: post-benchmarks runs: using: composite steps: - - uses: actions/upload-artifact@v3.1.3 + - name: Upload Benchmark Results artifact + uses: actions/upload-artifact@v4.0.0 with: - path: "${{ runner.temp }}/benchmark-results" + name: benchmark-results + path: "${{ runner.temp }}/benchmark-results/**" + if-no-files-found: error - name: Send benchmark report to visualisation run: |- set +e diff --git a/.github/actions/post-pmd-steps/action.yml b/.github/actions/post-pmd-steps/action.yml deleted file mode 100644 index 6c58fa49ec..0000000000 --- a/.github/actions/post-pmd-steps/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: post-pmd-steps -runs: - using: composite - steps: - - uses: actions/upload-artifact@v3.1.3 - with: - path: "${{ github.workspace }}/java/target/pmd.xml" - - uses: actions/upload-artifact@v3.1.3 - with: - path: "${{ github.workspace }}/java/target/site" diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml index 8318ff7656..4da4b641a1 100644 --- a/.github/actions/post-steps/action.yml +++ b/.github/actions/post-steps/action.yml @@ -1,25 +1,38 @@ name: post-steps +description: Steps that are taken after a RocksDB job +inputs: + artifact-prefix: + description: Prefix to append to the name of artifacts that are uploaded + required: true + default: "${{ github.job }}" runs: using: composite steps: - - uses: actions/upload-artifact@v3.1.3 + - name: Upload Test Results artifact + uses: actions/upload-artifact@v4.0.0 with: - path: "${{ runner.temp }}/test-results" - - uses: actions/upload-artifact@v3.1.3 + name: "${{ inputs.artifact-prefix }}-test-results" + path: "${{ runner.temp }}/test-results/**" + - name: Upload DB LOG file artifact + uses: actions/upload-artifact@v4.0.0 with: + name: "${{ inputs.artifact-prefix }}-db-log-file" path: LOG - - name: Compress Test Logs - run: tar -cvzf t.tar.gz t + - name: Copy Test Logs (on Failure) if: ${{ failure() }} + run: | + mkdir -p ${{ runner.temp }}/failure-test-logs + cp -r t/* ${{ runner.temp }}/failure-test-logs shell: bash - - uses: actions/upload-artifact@v3.1.3 + - name: Upload Test Logs (on Failure) artifact + uses: actions/upload-artifact@v4.0.0 with: - path: t.tar.gz - - run: |- - mkdir -p ${{ runner.temp }}/core_dumps - cp core.* ${{ runner.temp }}/core_dumps - if: ${{ failure() }} - shell: bash - - uses: actions/upload-artifact@v3.1.3 + name: "${{ inputs.artifact-prefix }}-failure-test-logs" + path: ${{ runner.temp }}/failure-test-logs/** + if-no-files-found: ignore + - name: Upload Core Dumps artifact + uses: actions/upload-artifact@v4.0.0 with: - path: "${{ runner.temp }}/core_dumps" + name: "${{ inputs.artifact-prefix }}-core-dumps" + path: "core.*" + if-no-files-found: ignore diff --git a/.github/workflows/pr-jobs.yml b/.github/workflows/pr-jobs.yml index ee769177da..1eee6a0cc6 100644 --- a/.github/workflows/pr-jobs.yml +++ b/.github/workflows/pr-jobs.yml @@ -324,8 +324,9 @@ jobs: - name: compress test report run: tar -cvzf scan_build_report.tar.gz scan_build_report if: failure() - - uses: actions/upload-artifact@v3.1.3 + - uses: actions/upload-artifact@v4.0.0 with: + name: scan-build-report path: scan_build_report.tar.gz build-linux-unity-and-headers: if: ${{ github.repository_owner == 'facebook' }} @@ -461,10 +462,20 @@ jobs: runs-on: labels: 4-core-ubuntu container: - image: zjay437/rocksdb:0.6 + image: evolvedbinary/rocksjava:centos6_x64-be options: --shm-size=16gb steps: - - uses: actions/checkout@v4.1.0 + # The docker image is intentionally based on an OS that has an older GLIBC version. + # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. + - name: Checkout + env: + GH_TOKEN: ${{ github.token }} + run: | + chown `whoami` . || true + git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . + git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} + git checkout --progress --force ${{ github.ref }} + git log -1 --format='%H' - uses: "./.github/actions/pre-steps" - name: Set Java Environment run: |- @@ -472,8 +483,8 @@ jobs: which java && java -version which javac && javac -version - name: Test RocksDBJava - run: make V=1 J=8 -j8 jtest - - uses: "./.github/actions/post-steps" + run: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest' + # NOTE: post-steps skipped because of compatibility issues with docker image build-linux-java-static: if: ${{ github.repository_owner == 'facebook' }} runs-on: @@ -482,9 +493,8 @@ jobs: image: evolvedbinary/rocksjava:centos6_x64-be options: --shm-size=16gb steps: - # The docker image is based on such an old OS that it has a GLIBC - # incompatibility with actions/checkout and other actions. Thus we - # implement a manual checkout step. + # The docker image is intentionally based on an OS that has an older GLIBC version. + # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. - name: Checkout env: GH_TOKEN: ${{ github.token }} @@ -574,6 +584,9 @@ jobs: if: ${{ github.repository_owner == 'facebook' }} runs-on: labels: 4-core-ubuntu + container: + image: evolvedbinary/rocksjava:rockylinux8_x64-be + options: --shm-size=16gb steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/install-maven" @@ -585,4 +598,11 @@ jobs: which javac && javac -version - name: PMD RocksDBJava run: make V=1 J=8 -j8 jpmd - - uses: "./.github/actions/post-pmd-steps" + - uses: actions/upload-artifact@v4.0.0 + with: + name: pmd-report + path: "${{ github.workspace }}/java/target/pmd.xml" + - uses: actions/upload-artifact@v4.0.0 + with: + name: maven-site + path: "${{ github.workspace }}/java/target/site" \ No newline at end of file