474 lines
14 KiB
Python
474 lines
14 KiB
Python
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from glob import glob
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
import nox
|
|
|
|
nox.options.sessions = ["test", "clippy", "fmt"]
|
|
|
|
|
|
PYO3_DIR = Path(__file__).parent
|
|
|
|
|
|
@nox.session(venv_backend="none")
|
|
def test(session: nox.Session) -> None:
|
|
test_rust(session)
|
|
test_py(session)
|
|
|
|
|
|
@nox.session(name="test-rust", venv_backend="none")
|
|
def test_rust(session: nox.Session):
|
|
_run_cargo_test(session, package="pyo3-build-config")
|
|
_run_cargo_test(session, package="pyo3-macros-backend")
|
|
_run_cargo_test(session, package="pyo3-macros")
|
|
_run_cargo_test(session, package="pyo3-ffi")
|
|
|
|
_run_cargo_test(session)
|
|
_run_cargo_test(session, features="abi3")
|
|
if not "skip-full" in session.posargs:
|
|
_run_cargo_test(session, features="full")
|
|
_run_cargo_test(session, features="abi3 full")
|
|
|
|
|
|
@nox.session(name="test-py", venv_backend="none")
|
|
def test_py(session: nox.Session) -> None:
|
|
_run(session, "nox", "-f", "pytests/noxfile.py", external=True)
|
|
for example in glob("examples/*/noxfile.py"):
|
|
_run(session, "nox", "-f", example, external=True)
|
|
|
|
|
|
@nox.session(venv_backend="none")
|
|
def coverage(session: nox.Session) -> None:
|
|
session.env.update(_get_coverage_env())
|
|
_run(session, "cargo", "llvm-cov", "clean", "--workspace", external=True)
|
|
test(session)
|
|
_run(
|
|
session,
|
|
"cargo",
|
|
"llvm-cov",
|
|
"--package=pyo3",
|
|
"--package=pyo3-build-config",
|
|
"--package=pyo3-macros-backend",
|
|
"--package=pyo3-macros",
|
|
"--package=pyo3-ffi",
|
|
"report",
|
|
"--lcov",
|
|
"--output-path",
|
|
"coverage.lcov",
|
|
external=True,
|
|
)
|
|
|
|
|
|
@nox.session
|
|
def fmt(session: nox.Session):
|
|
fmt_rust(session)
|
|
fmt_py(session)
|
|
|
|
|
|
@nox.session(name="fmt-rust", venv_backend="none")
|
|
def fmt_rust(session: nox.Session):
|
|
_run(session, "cargo", "fmt", "--all", "--check", external=True)
|
|
|
|
|
|
@nox.session(name="fmt-py")
|
|
def fmt_py(session: nox.Session):
|
|
session.install("black==22.3.0")
|
|
_run(session, "black", ".", "--check")
|
|
|
|
|
|
@nox.session(venv_backend="none")
|
|
def clippy(session: nox.Session) -> None:
|
|
for feature_set in ["full", "abi3 full"]:
|
|
_run(
|
|
session,
|
|
"cargo",
|
|
"clippy",
|
|
f"--features={feature_set}",
|
|
"--all-targets",
|
|
"--workspace",
|
|
"--",
|
|
"--deny=warnings",
|
|
external=True,
|
|
)
|
|
|
|
|
|
@nox.session(venv_backend="none")
|
|
def publish(session: nox.Session) -> None:
|
|
_run_cargo_publish(session, package="pyo3-build-config")
|
|
time.sleep(10)
|
|
_run_cargo_publish(session, package="pyo3-macros-backend")
|
|
time.sleep(10)
|
|
_run_cargo_publish(session, package="pyo3-macros")
|
|
time.sleep(10)
|
|
_run_cargo_publish(session, package="pyo3-ffi")
|
|
time.sleep(10)
|
|
_run_cargo_publish(session, package="pyo3")
|
|
|
|
|
|
@nox.session(venv_backend="none")
|
|
def contributors(session: nox.Session) -> None:
|
|
import requests
|
|
|
|
if len(session.posargs) < 1:
|
|
raise Exception("base commit positional argument missing")
|
|
|
|
base = session.posargs[0]
|
|
page = 1
|
|
|
|
head = "HEAD"
|
|
if len(session.posargs) == 2:
|
|
head = session.posargs[1]
|
|
|
|
if len(session.posargs) > 2:
|
|
raise Exception("too many arguments")
|
|
|
|
authors = set()
|
|
|
|
while True:
|
|
resp = requests.get(
|
|
f"https://api.github.com/repos/PyO3/pyo3/compare/{base}...{head}",
|
|
params={"page": page, "per_page": 100},
|
|
)
|
|
|
|
body = resp.json()
|
|
|
|
if resp.status_code != 200:
|
|
raise Exception(
|
|
f"failed to retrieve commits: {resp.status_code} {body['message']}"
|
|
)
|
|
|
|
for commit in body["commits"]:
|
|
try:
|
|
authors.add(commit["author"]["login"])
|
|
except:
|
|
continue
|
|
|
|
if "next" in resp.links:
|
|
page += 1
|
|
else:
|
|
break
|
|
|
|
authors = sorted(list(authors), key=lambda author: author.lower())
|
|
|
|
for author in authors:
|
|
print(f"@{author}")
|
|
|
|
|
|
class EmscriptenInfo:
|
|
def __init__(self):
|
|
self.emscripten_dir = PYO3_DIR / "emscripten"
|
|
self.builddir = PYO3_DIR / ".nox/emscripten"
|
|
self.builddir.mkdir(exist_ok=True, parents=True)
|
|
|
|
self.pyversion = sys.version.split()[0]
|
|
self.pymajor, self.pyminor, self.pymicro = self.pyversion.split(".")
|
|
self.pymicro, self.pydev = re.match(
|
|
"([0-9]*)([^0-9].*)?", self.pymicro
|
|
).groups()
|
|
if self.pydev is None:
|
|
self.pydev = ""
|
|
|
|
self.pymajorminor = f"{self.pymajor}.{self.pyminor}"
|
|
self.pymajorminormicro = f"{self.pymajorminor}.{self.pymicro}"
|
|
|
|
|
|
@nox.session(name="build-emscripten", venv_backend="none")
|
|
def build_emscripten(session: nox.Session):
|
|
info = EmscriptenInfo()
|
|
_run(
|
|
session,
|
|
"make",
|
|
"-C",
|
|
str(info.emscripten_dir),
|
|
f"BUILDROOT={info.builddir}",
|
|
f"PYMAJORMINORMICRO={info.pymajorminormicro}",
|
|
f"PYPRERELEASE={info.pydev}",
|
|
external=True,
|
|
)
|
|
|
|
|
|
@nox.session(name="test-emscripten", venv_backend="none")
|
|
def test_emscripten(session: nox.Session):
|
|
info = EmscriptenInfo()
|
|
|
|
libdir = info.builddir / f"install/Python-{info.pyversion}/lib"
|
|
pythonlibdir = libdir / f"python{info.pymajorminor}"
|
|
|
|
target = "wasm32-unknown-emscripten"
|
|
|
|
session.env["CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER"] = "python " + str(
|
|
info.emscripten_dir / "runner.py"
|
|
)
|
|
session.env["RUSTFLAGS"] = " ".join(
|
|
[
|
|
f"-L native={libdir}",
|
|
"-C link-arg=--preload-file",
|
|
f"-C link-arg={pythonlibdir}@/lib/python{info.pymajorminor}",
|
|
f"-C link-arg=-lpython{info.pymajorminor}",
|
|
"-C link-arg=-lexpat",
|
|
"-C link-arg=-lmpdec",
|
|
"-C link-arg=-lz",
|
|
"-C link-arg=-lbz2",
|
|
"-C link-arg=-sALLOW_MEMORY_GROWTH=1",
|
|
]
|
|
)
|
|
session.env["CARGO_BUILD_TARGET"] = target
|
|
session.env["PYO3_CROSS_LIB_DIR"] = pythonlibdir
|
|
_run(session, "rustup", "target", "add", target, "--toolchain", "stable")
|
|
_run(
|
|
session,
|
|
"bash",
|
|
"-c",
|
|
f"source {info.builddir/'emsdk/emsdk_env.sh'} && cargo test",
|
|
)
|
|
|
|
|
|
@nox.session(name="build-guide", venv_backend="none")
|
|
def build_guide(session: nox.Session):
|
|
_run(session, "mdbook", "build", "-d", "../target/guide", "guide", *session.posargs)
|
|
|
|
|
|
@nox.session(name="format-guide", venv_backend="none")
|
|
def format_guide(session: nox.Session):
|
|
fence_line = "//! ```\n"
|
|
|
|
for path in Path("guide").glob("**/*.md"):
|
|
session.log("Working on %s", path)
|
|
content = path.read_text()
|
|
|
|
lines = iter(path.read_text().splitlines(True))
|
|
new_lines = []
|
|
|
|
for line in lines:
|
|
new_lines.append(line)
|
|
if not re.search("```rust(,.*)?$", line):
|
|
continue
|
|
|
|
# Found a code block fence, gobble up its lines and write to temp. file
|
|
prefix = line[: line.index("```")]
|
|
with tempfile.NamedTemporaryFile("w", delete=False) as file:
|
|
tempname = file.name
|
|
file.write(fence_line)
|
|
for line in lines:
|
|
if line == prefix + "```\n":
|
|
break
|
|
file.write(("//! " + line[len(prefix) :]).rstrip() + "\n")
|
|
file.write(fence_line)
|
|
|
|
# Format it (needs nightly rustfmt for `format_code_in_doc_comments`)
|
|
_run(
|
|
session,
|
|
"rustfmt",
|
|
"+nightly",
|
|
"--config",
|
|
"format_code_in_doc_comments=true",
|
|
"--config",
|
|
"reorder_imports=false",
|
|
tempname,
|
|
)
|
|
|
|
# Re-read the formatted file, add its lines, and delete it
|
|
with open(tempname, "r") as file:
|
|
for line in file:
|
|
if line == fence_line:
|
|
continue
|
|
new_lines.append((prefix + line[4:]).rstrip() + "\n")
|
|
os.unlink(tempname)
|
|
|
|
new_lines.append(prefix + "```\n")
|
|
|
|
path.write_text("".join(new_lines))
|
|
|
|
|
|
@nox.session(name="address-sanitizer", venv_backend="none")
|
|
def address_sanitizer(session: nox.Session):
|
|
_run(
|
|
session,
|
|
"cargo",
|
|
"+nightly",
|
|
"test",
|
|
"--release",
|
|
"-Zbuild-std",
|
|
f"--target={_get_rust_target()}",
|
|
"--",
|
|
"--test-threads=1",
|
|
env={
|
|
"RUSTFLAGS": "-Zsanitizer=address",
|
|
"RUSTDOCFLAGS": "-Zsanitizer=address",
|
|
"ASAN_OPTIONS": "detect_leaks=0",
|
|
},
|
|
external=True,
|
|
)
|
|
|
|
|
|
@nox.session(name="check-changelog")
|
|
def check_changelog(session: nox.Session):
|
|
event_path = os.environ.get("GITHUB_EVENT_PATH")
|
|
if event_path is None:
|
|
session.error("Can only check changelog on github actions")
|
|
|
|
with open(event_path) as event_file:
|
|
event = json.load(event_file)
|
|
|
|
if event["pull_request"]["title"].startswith("release:"):
|
|
session.skip("PR title starts with release")
|
|
|
|
for label in event["pull_request"]["labels"]:
|
|
if label["name"] == "CI-skip-changelog":
|
|
session.skip("CI-skip-changelog label applied")
|
|
|
|
issue_number = event["pull_request"]["number"]
|
|
|
|
newsfragments = PYO3_DIR / "newsfragments"
|
|
|
|
fragments = tuple(
|
|
filter(
|
|
Path.exists,
|
|
(
|
|
newsfragments / f"{issue_number}.{change_type}.md"
|
|
for change_type in ("packaging", "added", "changed", "removed", "fixed")
|
|
),
|
|
)
|
|
)
|
|
|
|
if not fragments:
|
|
session.error(
|
|
"Changelog entry not found, please add one (or more) to `newsfragments` directory. For more information see https://github.com/PyO3/pyo3/blob/main/Contributing.md#documenting-changes"
|
|
)
|
|
|
|
print("Found newsfragments:")
|
|
for fragment in fragments:
|
|
print(fragment.name)
|
|
|
|
|
|
@nox.session(name="set-minimal-package-versions")
|
|
def set_minimal_package_versions(session: nox.Session):
|
|
# run cargo update first to ensure that everything is at highest
|
|
# possible version, so that this matches what CI will resolve to.
|
|
_run(session, "cargo", "update", external=True)
|
|
|
|
_run_cargo_set_package_version(session, "indexmap", "1.6.2")
|
|
_run_cargo_set_package_version(session, "hashbrown:0.12.3", "0.9.1")
|
|
_run_cargo_set_package_version(session, "plotters", "0.3.1")
|
|
_run_cargo_set_package_version(session, "plotters-svg", "0.3.1")
|
|
_run_cargo_set_package_version(session, "plotters-backend", "0.3.2")
|
|
_run_cargo_set_package_version(session, "bumpalo", "3.10.0")
|
|
_run_cargo_set_package_version(session, "once_cell", "1.14.0")
|
|
_run_cargo_set_package_version(session, "rayon", "1.5.3")
|
|
_run_cargo_set_package_version(session, "rayon-core", "1.9.3")
|
|
|
|
# 1.15.0 depends on hermit-abi 0.2.6 which has edition 2021 and breaks 1.48.0
|
|
_run_cargo_set_package_version(session, "num_cpus", "1.14.0")
|
|
_run_cargo_set_package_version(
|
|
session, "num_cpus", "1.14.0", project="examples/word-count"
|
|
)
|
|
|
|
projects = (
|
|
None,
|
|
"examples/decorator",
|
|
"examples/maturin-starter",
|
|
"examples/setuptools-rust-starter",
|
|
"examples/word-count",
|
|
)
|
|
for project in projects:
|
|
_run_cargo_set_package_version(
|
|
session, "parking_lot", "0.11.0", project=project
|
|
)
|
|
_run_cargo_set_package_version(session, "once_cell", "1.14.0", project=project)
|
|
|
|
_run_cargo_set_package_version(
|
|
session, "rayon", "1.5.3", project="examples/word-count"
|
|
)
|
|
_run_cargo_set_package_version(
|
|
session, "rayon-core", "1.9.3", project="examples/word-count"
|
|
)
|
|
|
|
# As a smoke test, cargo metadata solves all dependencies, so
|
|
# will break if any crates rely on cargo features not
|
|
# supported on MSRV
|
|
_run(session, "cargo", "metadata", silent=True, external=True)
|
|
|
|
|
|
def _get_rust_target() -> str:
|
|
output = _get_output("rustc", "-vV")
|
|
|
|
for line in output.splitlines():
|
|
if line.startswith(_HOST_LINE_START):
|
|
return line[len(_HOST_LINE_START) :].strip()
|
|
|
|
|
|
_HOST_LINE_START = "host: "
|
|
|
|
|
|
def _get_coverage_env() -> Dict[str, str]:
|
|
env = {}
|
|
output = _get_output("cargo", "llvm-cov", "show-env")
|
|
|
|
for line in output.strip().splitlines():
|
|
(key, value) = line.split("=", maxsplit=1)
|
|
env[key] = value.strip('"')
|
|
|
|
# Ensure that examples/ and pytests/ all build to the correct target directory to collect
|
|
# coverage artifacts.
|
|
env["CARGO_TARGET_DIR"] = env["CARGO_LLVM_COV_TARGET_DIR"]
|
|
|
|
return env
|
|
|
|
|
|
def _run(session: nox.Session, *args: str, **kwargs: Any) -> None:
|
|
"""Wrapper for _run(session, which creates nice groups on GitHub Actions."""
|
|
if "GITHUB_ACTIONS" in os.environ:
|
|
# Insert ::group:: at the start of nox's command line output
|
|
print("::group::", end="", flush=True, file=sys.stderr)
|
|
session.run(*args, **kwargs)
|
|
if "GITHUB_ACTIONS" in os.environ:
|
|
print("::endgroup::", file=sys.stderr)
|
|
|
|
|
|
def _run_cargo_test(
|
|
session: nox.Session,
|
|
*,
|
|
package: Optional[str] = None,
|
|
features: Optional[str] = None,
|
|
) -> None:
|
|
command = ["cargo"]
|
|
if "careful" in session.posargs:
|
|
command.append("careful")
|
|
command.append("test")
|
|
if "release" in session.posargs:
|
|
command.append("--release")
|
|
if package:
|
|
command.append(f"--package={package}")
|
|
if features:
|
|
command.append(f"--features={features}")
|
|
|
|
_run(session, *command, external=True)
|
|
|
|
|
|
def _run_cargo_publish(session: nox.Session, *, package: str) -> None:
|
|
_run(session, "cargo", "publish", f"--package={package}", external=True)
|
|
|
|
|
|
def _run_cargo_set_package_version(
|
|
session: nox.Session,
|
|
package: str,
|
|
version: str,
|
|
*,
|
|
project: Optional[str] = None,
|
|
) -> None:
|
|
command = ["cargo", "update", "-p", package, "--precise", version]
|
|
if project:
|
|
command.append(f"--manifest-path={project}/Cargo.toml")
|
|
_run(session, *command, external=True)
|
|
|
|
|
|
def _get_output(*args: str) -> str:
|
|
return subprocess.run(args, capture_output=True, text=True, check=True).stdout
|