From cda034b29ece5112593779e80679a38682a2ae0b Mon Sep 17 00:00:00 2001 From: Nettika Date: Mon, 16 Feb 2026 12:28:25 -0800 Subject: [PATCH] Migrate to wedges --- README.md | 17 ++- default.nix | 2 +- mypy-0.780.nix | 51 ------- mypy-extensions-0.4.4.nix | 23 ---- oils-for-unix-0.37.0.nix => package.nix | 91 +++++++------ requirements.txt | 8 ++ typed-ast-1.5.5.nix | 27 ---- wedges.nix | 174 ++++++++++++++++++++++++ 8 files changed, 242 insertions(+), 151 deletions(-) delete mode 100644 mypy-0.780.nix delete mode 100644 mypy-extensions-0.4.4.nix rename oils-for-unix-0.37.0.nix => package.nix (63%) create mode 100644 requirements.txt delete mode 100644 typed-ast-1.5.5.nix create mode 100644 wedges.nix diff --git a/README.md b/README.md index cf52955..ce335fd 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,19 @@ # oils-for-unix from source ```bash -NIXPKGS_ALLOW_INSECURE=1 nix-build . +nix-build . ``` -The build uses Python 2 for legacy build scripts. Python 2, marked insecure in nixpkgs, is not present in the final binaries. +Builds OSH and YSH shell interpreters from git source using upstream-style wedges. ## Files -- `oils-for-unix-0.37.0.nix` - from-source Oils -- `mypy-0.780.nix` - mypy 0.780 (last version to support --py2) built without mypyc -- `mypy-extensions-0.4.4.nix` - mypy-extensions pinned to <0.5.0 for mypy 0.780 compatibility -- `typed-ast-1.5.5.nix` - typed-ast for Python 2 AST parsing (removed from nixpkgs) +- `package.nix` - Main oils-for-unix derivation +- `wedges.nix` - Python 2.7.18, Python 3.10.4, and mypy 0.780 built from source +- `requirements.txt` - Pinned pip dependencies for mypy + +## Why wedges? + +The build uses Python 2 for legacy build scripts and mypy 0.780 (last version with `--py2` support). Rather than depending on nixpkgs Python packages (which have compatibility issues), we build Python from official tarballs - mirroring oils' own `build/deps.sh` wedge system. + +Python is only used at build time; final binaries are native C++. diff --git a/default.nix b/default.nix index c3a78eb..343644c 100644 --- a/default.nix +++ b/default.nix @@ -2,4 +2,4 @@ pkgs ? import { }, }: -pkgs.callPackage ./oils-for-unix-0.37.0.nix { } +pkgs.callPackage ./package.nix { } diff --git a/mypy-0.780.nix b/mypy-0.780.nix deleted file mode 100644 index 046a2b7..0000000 --- a/mypy-0.780.nix +++ /dev/null @@ -1,51 +0,0 @@ -{ - lib, - fetchFromGitHub, - python3, - # Injected dependencies - mypy-extensions, - typed-ast, -}: - -# mypy 0.780 is the last version with --py2 support -# Built without mypyc so mycpp can extend its classes -python3.pkgs.buildPythonPackage (finalAttrs: { - pname = "mypy"; - version = "0.780"; - - pyproject = true; - build-system = [ python3.pkgs.setuptools ]; - - src = fetchFromGitHub { - owner = "python"; - repo = "mypy"; - rev = "v${finalAttrs.version}"; - # Fetch submodules to include typeshed (required for type stubs) - fetchSubmodules = true; - hash = "sha256-czwCx6ZjCu3CrVmbI6NbstzWM0GvuPTWJiiUhXSznu4="; - }; - - # Do not use mypyc to compile. - # Rather, build interpreted so mycpp can extend its classes. - preBuild = '' - export MYPY_USE_MYPYC=0 - ''; - - # Skip tests to speed up build - doCheck = false; - - # Skip runtime dependency check since we're using pinned older versions - pythonRuntimeDepsCheck = false; - dontCheckRuntimeDeps = true; - - dependencies = [ - mypy-extensions - typed-ast - python3.pkgs.typing-extensions - ]; - - # Ensure we're not pulling in a compiled version - pythonImportsCheck = [ "mypy" ]; - - meta.license = lib.licenses.mit; -}) diff --git a/mypy-extensions-0.4.4.nix b/mypy-extensions-0.4.4.nix deleted file mode 100644 index ac7a8bd..0000000 --- a/mypy-extensions-0.4.4.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ - lib, - fetchFromGitHub, - python3, -}: - -# mypy 0.780 requires mypy-extensions <0.5.0 -python3.pkgs.buildPythonPackage (finalAttrs: { - pname = "mypy-extensions"; - version = "0.4.4"; - pyproject = true; - build-system = [ python3.pkgs.setuptools ]; - - src = fetchFromGitHub { - owner = "python"; - repo = "mypy_extensions"; - rev = finalAttrs.version; - hash = "sha256-eZdEJJA6HOf1iQLV9H9qqETEByCgIH6oy+wwffGJVuQ="; - }; - - pythonImportsCheck = [ "mypy_extensions" ]; - meta.license = lib.licenses.mit; -}) diff --git a/oils-for-unix-0.37.0.nix b/package.nix similarity index 63% rename from oils-for-unix-0.37.0.nix rename to package.nix index a3a82ee..2c475db 100644 --- a/oils-for-unix-0.37.0.nix +++ b/package.nix @@ -2,67 +2,56 @@ lib, stdenv, fetchFromGitHub, - symlinkJoin, callPackage, - # Build tools - python2, # Legacy build scripts. Marked insecure but not in final binary - python310, # Newest version that supports mypy 0.780's typeshed - re2c, + symlinkJoin, + pkg-config, ninja, git, + re2c, cmark, which, time, readline, - # Python dependencies for mycpp - mypy-extensions ? callPackage ./mypy-extensions-0.4.4.nix { python3 = python310; }, - typed-ast ? callPackage ./typed-ast-1.5.5.nix { python3 = python310; }, - mypy-for-mycpp ? callPackage ./mypy-0.780.nix { - python3 = python310; - inherit mypy-extensions typed-ast; - }, + ncurses, + zlib, + libffi, }: let - python = python310.withPackages (ps: [ - mypy-for-mycpp - ps.typing-extensions - ]); - - # Configure script expects readline lib and headers at the same prefix - readline-all = symlinkJoin { - name = "readline-all"; - paths = [ - readline - readline.dev - ]; - }; - -in -stdenv.mkDerivation (finalAttrs: { - pname = "oils-for-unix"; version = "0.37.0"; src = fetchFromGitHub { owner = "oils-for-unix"; repo = "oils"; - # Reference by release branch - rev = "release/${finalAttrs.version}"; + rev = "release/${version}"; hash = "sha256-d2i2P8ZiGb+FYzZIzs0pY2gIRQGGuenLbxrGhafVxVc="; }; + wedges = callPackage ./wedges.nix { }; + +in +stdenv.mkDerivation (finalAttrs: { + pname = "oils-for-unix"; + inherit version src; + nativeBuildInputs = [ - python2 - python - re2c + pkg-config ninja git + re2c cmark which time + wedges.python2Wedge + wedges.python3Wedge ]; - buildInputs = [ readline ]; + buildInputs = [ + readline + ncurses + zlib + libffi + ]; postPatch = '' patchShebangs . @@ -70,7 +59,6 @@ stdenv.mkDerivation (finalAttrs: { substituteInPlace build/dynamic-deps.sh \ --replace-quiet '/usr/bin/env' "$(command -v env)" - # _PyVerify_fd is Windows-only substituteInPlace pyext/posixmodule.c \ --replace-quiet '_PyVerify_fd(fd)' '1' @@ -81,16 +69,23 @@ stdenv.mkDerivation (finalAttrs: { configurePhase = '' runHook preConfigure + ./configure \ --datarootdir=$out \ --with-readline \ - --readline=${readline-all} + --readline=${wedges.readline-all} + runHook postConfigure ''; buildPhase = '' runHook preBuild - . build/dev-shell.sh + + export PATH="${wedges.python2Wedge}/bin:${wedges.python3Wedge}/bin:$PATH" + export PATH="${wedges.mypyWedge}/bin:$PATH" + export PYTHONPATH="${wedges.mypyWedge}/lib/python3.10/site-packages:.:vendor/" + export LD_LIBRARY_PATH="${wedges.python2Wedge}/lib:${wedges.python3Wedge}/lib:''${LD_LIBRARY_PATH:-}" + build/py.sh configure-for-dev build/stamp.sh write-git-commit build/py.sh py-source @@ -98,33 +93,42 @@ stdenv.mkDerivation (finalAttrs: { build/doc.sh all-ref ./NINJA-config.sh ninja _bin/cxx-opt/oils-for-unix + runHook postBuild ''; installPhase = '' runHook preInstall + mkdir -p $out/bin install -m755 _bin/cxx-opt/oils-for-unix $out/bin/oils-for-unix ln -s oils-for-unix $out/bin/osh ln -s oils-for-unix $out/bin/ysh + runHook postInstall ''; - # Suppress warnings from older C code in vendor dependencies env.NIX_CFLAGS_COMPILE = lib.optionalString stdenv.cc.isClang "-Wno-error=incompatible-function-pointer-types"; meta = { + description = "Oils is our upgrade path from bash to a better language and runtime"; + homepage = "https://www.oilshell.org/"; license = lib.licenses.asl20; - platforms = lib.platforms.all; + platforms = lib.platforms.unix; mainProgram = "osh"; }; - passthru = + passthru = { + inherit wedges; + + withSrc = newSrc: finalAttrs.finalPackage.overrideAttrs { src = newSrc; }; + } + // ( let mkShell = shellName: symlinkJoin { - name = "oils-for-unix-${shellName}-${finalAttrs.version}"; + name = "oils-for-unix-${shellName}-${version}"; paths = [ finalAttrs.finalPackage ]; passthru.shellPath = "/bin/${shellName}"; meta = finalAttrs.meta // { @@ -135,5 +139,6 @@ stdenv.mkDerivation (finalAttrs: { { osh = mkShell "osh"; ysh = mkShell "ysh"; - }; + } + ); }) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b60814a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +pip==26.0.1 +setuptools==82.0.0 +wheel==0.46.3 +packaging==24.2 +typing_extensions==4.15.0 +mypy_extensions==0.4.3 +typed_ast==1.5.5 +mypy==0.780 diff --git a/typed-ast-1.5.5.nix b/typed-ast-1.5.5.nix deleted file mode 100644 index d823aa7..0000000 --- a/typed-ast-1.5.5.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ - lib, - fetchFromGitHub, - python3, -}: - -# typed-ast was removed from nixpkgs but is needed for mypy 0.780 -python3.pkgs.buildPythonPackage (finalAttrs: { - pname = "typed-ast"; - version = "1.5.5"; - pyproject = true; - build-system = [ python3.pkgs.setuptools ]; - - src = fetchFromGitHub { - owner = "python"; - repo = "typed_ast"; - rev = finalAttrs.version; - hash = "sha256-A/FA6ngu8/bbpKW9coJ7unm9GQezGuDhgBWjOhAxm2o="; - }; - - # GCC 15 uses C23 by default where false/true/bool are keywords - # Force C11 to build this legacy package - env.NIX_CFLAGS_COMPILE = "-std=c11"; - - pythonImportsCheck = [ "typed_ast" ]; - meta.license = lib.licenses.asl20; -}) diff --git a/wedges.nix b/wedges.nix new file mode 100644 index 0000000..2d03498 --- /dev/null +++ b/wedges.nix @@ -0,0 +1,174 @@ +# Wedge derivations for oils-for-unix +# +# These mirror oils' upstream wedge system (build/deps.sh): +# - Python 2.7 and 3.10 built from official tarballs +# - mypy and dependencies downloaded via pip (FOD) +# - mypy installed in a venv from cached packages + +{ + stdenv, + fetchFromGitHub, + fetchurl, + symlinkJoin, + pkg-config, + cacert, + openssl, + zlib, + libffi, + readline, + ncurses, + bzip2, + xz, + gdbm, + sqlite, +}: + +let + mkPythonWedge = + { + version, + hash, + extraConfigureFlags ? [ ], + }: + stdenv.mkDerivation (finalAttrs: { + pname = "oils-python-wedge"; + inherit version; + + src = fetchurl { + url = "https://www.python.org/ftp/python/${finalAttrs.version}/Python-${finalAttrs.version}.tar.xz"; + inherit hash; + }; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ + readline + ncurses + zlib + bzip2 + openssl + libffi + xz + gdbm + sqlite + ]; + + env = { + CPPFLAGS = "-I${zlib.dev}/include -I${libffi.dev}/include"; + LDFLAGS = "-L${zlib}/lib -L${libffi}/lib -Wl,-rpath,${placeholder "out"}/lib"; + PKG_CONFIG_PATH = "${libffi.dev}/lib/pkgconfig:${zlib.dev}/lib/pkgconfig:${openssl.dev}/lib/pkgconfig"; + LIBFFI_INCLUDEDIR = "${libffi.dev}/include"; + LIBFFI_LIBDIR = "${libffi}/lib"; + }; + + configureFlags = [ + "--enable-shared" + "--with-system-ffi" + ] ++ extraConfigureFlags; + + doCheck = false; + enableParallelBuilding = true; + }); + + python2Wedge = mkPythonWedge { + version = "2.7.18"; + hash = "sha256-tiwOeTdVHQzAK4/Vyw9UT5QFuvyaVNOAjtRZSBLt70M="; + extraConfigureFlags = [ + "--enable-unicode=ucs4" + "--with-ensurepip=no" + ]; + }; + + python3Wedge = mkPythonWedge { + version = "3.10.4"; + hash = "sha256-gL+SX1cdpDazUhCIbPefbrX6XWxXExa3NWg0NFH3ehk="; + extraConfigureFlags = [ + "--with-ensurepip=install" + ]; + }; + + py3LibsCache = stdenv.mkDerivation { + name = "oils-py3-libs-cache"; + + nativeBuildInputs = [ + python3Wedge + cacert + ]; + + dontUnpack = true; + + buildPhase = '' + export HOME=$TMPDIR + export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt + + mkdir -p $out + + ${python3Wedge}/bin/pip3 install --upgrade pip setuptools wheel + + # Download exact versions (see requirements.txt) + ${python3Wedge}/bin/pip3 download -d $out --only-binary=:all: --no-deps \ + -r ${./requirements.txt} + ''; + + dontInstall = true; + + outputHashMode = "recursive"; + outputHash = "sha256-dTO5D4vr42A7Pui0i8ggt8rD6xQplG17rGB76M4ntRY="; + }; + + mypyWedge = stdenv.mkDerivation (finalAttrs: { + pname = "oils-mypy-wedge"; + version = "0.780"; + + src = fetchFromGitHub { + owner = "python"; + repo = "mypy"; + rev = "v${finalAttrs.version}"; + hash = "sha256-czwCx6ZjCu3CrVmbI6NbstzWM0GvuPTWJiiUhXSznu4="; + fetchSubmodules = true; + }; + + nativeBuildInputs = [ python3Wedge ]; + + buildPhase = '' + runHook preBuild + + export HOME=$TMPDIR + + ${python3Wedge}/bin/python3 -m venv $out + + # Install all packages from cached wheels (--no-deps to bypass version conflicts) + $out/bin/pip install \ + --no-index \ + --find-links=${py3LibsCache} \ + --no-deps \ + setuptools wheel packaging \ + typing_extensions mypy_extensions typed_ast \ + mypy + + # Copy mypy source (mycpp extends mypy classes) + cp -r . $out/mypy-src + + runHook postBuild + ''; + + dontInstall = true; + dontFixup = true; + }); + + readline-all = symlinkJoin { + name = "readline-all"; + paths = [ + readline + readline.dev + ]; + }; + +in +{ + inherit + python2Wedge + python3Wedge + mypyWedge + readline-all + ; +}