Migrate to wedges

README.md

@@ -1,14 +1,19 @@
 # oils-for-unix from source
 
 ```bash
-NIXPKGS_ALLOW_INSECURE=1 nix-build .
+nix-build .
 ```
 
-The build uses Python 2 for legacy build scripts. Python 2, marked insecure in nixpkgs, is not present in the final binaries.
+Builds OSH and YSH shell interpreters from git source using upstream-style wedges.
 
 ## Files
 
-- `oils-for-unix-0.37.0.nix` - from-source Oils
+- `package.nix` - Main oils-for-unix derivation
-- `mypy-0.780.nix` - mypy 0.780 (last version to support --py2) built without mypyc
+- `wedges.nix` - Python 2.7.18, Python 3.10.4, and mypy 0.780 built from source
-- `mypy-extensions-0.4.4.nix` - mypy-extensions pinned to <0.5.0 for mypy 0.780 compatibility
+- `requirements.txt` - Pinned pip dependencies for mypy
-- `typed-ast-1.5.5.nix` - typed-ast for Python 2 AST parsing (removed from nixpkgs)
+
+## Why wedges?
+
+The build uses Python 2 for legacy build scripts and mypy 0.780 (last version with `--py2` support). Rather than depending on nixpkgs Python packages (which have compatibility issues), we build Python from official tarballs - mirroring oils' own `build/deps.sh` wedge system.
+
+Python is only used at build time; final binaries are native C++.

default.nix

@@ -2,4 +2,4 @@
   pkgs ? import <nixpkgs> { },
 }:
 
-pkgs.callPackage ./oils-for-unix-0.37.0.nix { }
+pkgs.callPackage ./package.nix { }

mypy-0.780.nix

@@ -1,51 +0,0 @@
-{
-  lib,
-  fetchFromGitHub,
-  python3,
-  # Injected dependencies
-  mypy-extensions,
-  typed-ast,
-}:
-
-# mypy 0.780 is the last version with --py2 support
-# Built without mypyc so mycpp can extend its classes
-python3.pkgs.buildPythonPackage (finalAttrs: {
-  pname = "mypy";
-  version = "0.780";
-
-  pyproject = true;
-  build-system = [ python3.pkgs.setuptools ];
-
-  src = fetchFromGitHub {
-    owner = "python";
-    repo = "mypy";
-    rev = "v${finalAttrs.version}";
-    # Fetch submodules to include typeshed (required for type stubs)
-    fetchSubmodules = true;
-    hash = "sha256-czwCx6ZjCu3CrVmbI6NbstzWM0GvuPTWJiiUhXSznu4=";
-  };
-
-  # Do not use mypyc to compile.
-  # Rather, build interpreted so mycpp can extend its classes.
-  preBuild = ''
-    export MYPY_USE_MYPYC=0
-  '';
-
-  # Skip tests to speed up build
-  doCheck = false;
-
-  # Skip runtime dependency check since we're using pinned older versions
-  pythonRuntimeDepsCheck = false;
-  dontCheckRuntimeDeps = true;
-
-  dependencies = [
-    mypy-extensions
-    typed-ast
-    python3.pkgs.typing-extensions
-  ];
-
-  # Ensure we're not pulling in a compiled version
-  pythonImportsCheck = [ "mypy" ];
-
-  meta.license = lib.licenses.mit;
-})

mypy-extensions-0.4.4.nix

@@ -1,23 +0,0 @@
-{
-  lib,
-  fetchFromGitHub,
-  python3,
-}:
-
-# mypy 0.780 requires mypy-extensions <0.5.0
-python3.pkgs.buildPythonPackage (finalAttrs: {
-  pname = "mypy-extensions";
-  version = "0.4.4";
-  pyproject = true;
-  build-system = [ python3.pkgs.setuptools ];
-
-  src = fetchFromGitHub {
-    owner = "python";
-    repo = "mypy_extensions";
-    rev = finalAttrs.version;
-    hash = "sha256-eZdEJJA6HOf1iQLV9H9qqETEByCgIH6oy+wwffGJVuQ=";
-  };
-
-  pythonImportsCheck = [ "mypy_extensions" ];
-  meta.license = lib.licenses.mit;
-})

package.nix

@@ -2,67 +2,56 @@
   lib,
   stdenv,
   fetchFromGitHub,
-  symlinkJoin,
   callPackage,
-  # Build tools
+  symlinkJoin,
-  python2, # Legacy build scripts. Marked insecure but not in final binary
+  pkg-config,
-  python310, # Newest version that supports mypy 0.780's typeshed
-  re2c,
   ninja,
   git,
+  re2c,
   cmark,
   which,
   time,
   readline,
-  # Python dependencies for mycpp
+  ncurses,
-  mypy-extensions ? callPackage ./mypy-extensions-0.4.4.nix { python3 = python310; },
+  zlib,
-  typed-ast ? callPackage ./typed-ast-1.5.5.nix { python3 = python310; },
+  libffi,
-  mypy-for-mycpp ? callPackage ./mypy-0.780.nix {
-    python3 = python310;
-    inherit mypy-extensions typed-ast;
-  },
 }:
 
 let
-  python = python310.withPackages (ps: [
-    mypy-for-mycpp
-    ps.typing-extensions
-  ]);
-
-  # Configure script expects readline lib and headers at the same prefix
-  readline-all = symlinkJoin {
-    name = "readline-all";
-    paths = [
-      readline
-      readline.dev
-    ];
-  };
-
-in
-stdenv.mkDerivation (finalAttrs: {
-  pname = "oils-for-unix";
   version = "0.37.0";
 
   src = fetchFromGitHub {
     owner = "oils-for-unix";
     repo = "oils";
-    # Reference by release branch
+    rev = "release/${version}";
-    rev = "release/${finalAttrs.version}";
     hash = "sha256-d2i2P8ZiGb+FYzZIzs0pY2gIRQGGuenLbxrGhafVxVc=";
   };
 
+  wedges = callPackage ./wedges.nix { };
+
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "oils-for-unix";
+  inherit version src;
+
   nativeBuildInputs = [
-    python2
+    pkg-config
-    python
-    re2c
     ninja
     git
+    re2c
     cmark
     which
     time
+    wedges.python2Wedge
+    wedges.python3Wedge
   ];
 
-  buildInputs = [ readline ];
+  buildInputs = [
+    readline
+    ncurses
+    zlib
+    libffi
+  ];
 
   postPatch = ''
     patchShebangs .
@@ -70,7 +59,6 @@ stdenv.mkDerivation (finalAttrs: {
     substituteInPlace build/dynamic-deps.sh \
       --replace-quiet '/usr/bin/env' "$(command -v env)"
 
-    # _PyVerify_fd is Windows-only
     substituteInPlace pyext/posixmodule.c \
       --replace-quiet '_PyVerify_fd(fd)' '1'
 
@@ -81,16 +69,23 @@ stdenv.mkDerivation (finalAttrs: {
 
   configurePhase = ''
     runHook preConfigure
+
     ./configure \
       --datarootdir=$out \
       --with-readline \
-      --readline=${readline-all}
+      --readline=${wedges.readline-all}
+
     runHook postConfigure
   '';
 
   buildPhase = ''
     runHook preBuild
-    . build/dev-shell.sh
+
+    export PATH="${wedges.python2Wedge}/bin:${wedges.python3Wedge}/bin:$PATH"
+    export PATH="${wedges.mypyWedge}/bin:$PATH"
+    export PYTHONPATH="${wedges.mypyWedge}/lib/python3.10/site-packages:.:vendor/"
+    export LD_LIBRARY_PATH="${wedges.python2Wedge}/lib:${wedges.python3Wedge}/lib:''${LD_LIBRARY_PATH:-}"
+
     build/py.sh configure-for-dev
     build/stamp.sh write-git-commit
     build/py.sh py-source
@@ -98,33 +93,42 @@ stdenv.mkDerivation (finalAttrs: {
     build/doc.sh all-ref
     ./NINJA-config.sh
     ninja _bin/cxx-opt/oils-for-unix
+
     runHook postBuild
   '';
 
   installPhase = ''
     runHook preInstall
+
     mkdir -p $out/bin
     install -m755 _bin/cxx-opt/oils-for-unix $out/bin/oils-for-unix
     ln -s oils-for-unix $out/bin/osh
     ln -s oils-for-unix $out/bin/ysh
+
     runHook postInstall
   '';
 
-  # Suppress warnings from older C code in vendor dependencies
   env.NIX_CFLAGS_COMPILE = lib.optionalString stdenv.cc.isClang "-Wno-error=incompatible-function-pointer-types";
 
   meta = {
+    description = "Oils is our upgrade path from bash to a better language and runtime";
+    homepage = "https://www.oilshell.org/";
     license = lib.licenses.asl20;
-    platforms = lib.platforms.all;
+    platforms = lib.platforms.unix;
     mainProgram = "osh";
   };
 
-  passthru =
+  passthru = {
+    inherit wedges;
+
+    withSrc = newSrc: finalAttrs.finalPackage.overrideAttrs { src = newSrc; };
+  }
+  // (
     let
       mkShell =
         shellName:
         symlinkJoin {
-          name = "oils-for-unix-${shellName}-${finalAttrs.version}";
+          name = "oils-for-unix-${shellName}-${version}";
           paths = [ finalAttrs.finalPackage ];
           passthru.shellPath = "/bin/${shellName}";
           meta = finalAttrs.meta // {
@@ -135,5 +139,6 @@ stdenv.mkDerivation (finalAttrs: {
     {
       osh = mkShell "osh";
       ysh = mkShell "ysh";
-    };
+    }
+  );
 })

requirements.txt

@@ -0,0 +1,8 @@
+pip==26.0.1
+setuptools==82.0.0
+wheel==0.46.3
+packaging==24.2
+typing_extensions==4.15.0
+mypy_extensions==0.4.3
+typed_ast==1.5.5
+mypy==0.780

typed-ast-1.5.5.nix

@@ -1,27 +0,0 @@
-{
-  lib,
-  fetchFromGitHub,
-  python3,
-}:
-
-# typed-ast was removed from nixpkgs but is needed for mypy 0.780
-python3.pkgs.buildPythonPackage (finalAttrs: {
-  pname = "typed-ast";
-  version = "1.5.5";
-  pyproject = true;
-  build-system = [ python3.pkgs.setuptools ];
-
-  src = fetchFromGitHub {
-    owner = "python";
-    repo = "typed_ast";
-    rev = finalAttrs.version;
-    hash = "sha256-A/FA6ngu8/bbpKW9coJ7unm9GQezGuDhgBWjOhAxm2o=";
-  };
-
-  # GCC 15 uses C23 by default where false/true/bool are keywords
-  # Force C11 to build this legacy package
-  env.NIX_CFLAGS_COMPILE = "-std=c11";
-
-  pythonImportsCheck = [ "typed_ast" ];
-  meta.license = lib.licenses.asl20;
-})

wedges.nix

@@ -0,0 +1,174 @@
+# Wedge derivations for oils-for-unix
+#
+# These mirror oils' upstream wedge system (build/deps.sh):
+# - Python 2.7 and 3.10 built from official tarballs
+# - mypy and dependencies downloaded via pip (FOD)
+# - mypy installed in a venv from cached packages
+
+{
+  stdenv,
+  fetchFromGitHub,
+  fetchurl,
+  symlinkJoin,
+  pkg-config,
+  cacert,
+  openssl,
+  zlib,
+  libffi,
+  readline,
+  ncurses,
+  bzip2,
+  xz,
+  gdbm,
+  sqlite,
+}:
+
+let
+  mkPythonWedge =
+    {
+      version,
+      hash,
+      extraConfigureFlags ? [ ],
+    }:
+    stdenv.mkDerivation (finalAttrs: {
+      pname = "oils-python-wedge";
+      inherit version;
+
+      src = fetchurl {
+        url = "https://www.python.org/ftp/python/${finalAttrs.version}/Python-${finalAttrs.version}.tar.xz";
+        inherit hash;
+      };
+
+      nativeBuildInputs = [ pkg-config ];
+      buildInputs = [
+        readline
+        ncurses
+        zlib
+        bzip2
+        openssl
+        libffi
+        xz
+        gdbm
+        sqlite
+      ];
+
+      env = {
+        CPPFLAGS = "-I${zlib.dev}/include -I${libffi.dev}/include";
+        LDFLAGS = "-L${zlib}/lib -L${libffi}/lib -Wl,-rpath,${placeholder "out"}/lib";
+        PKG_CONFIG_PATH = "${libffi.dev}/lib/pkgconfig:${zlib.dev}/lib/pkgconfig:${openssl.dev}/lib/pkgconfig";
+        LIBFFI_INCLUDEDIR = "${libffi.dev}/include";
+        LIBFFI_LIBDIR = "${libffi}/lib";
+      };
+
+      configureFlags = [
+        "--enable-shared"
+        "--with-system-ffi"
+      ] ++ extraConfigureFlags;
+
+      doCheck = false;
+      enableParallelBuilding = true;
+    });
+
+  python2Wedge = mkPythonWedge {
+    version = "2.7.18";
+    hash = "sha256-tiwOeTdVHQzAK4/Vyw9UT5QFuvyaVNOAjtRZSBLt70M=";
+    extraConfigureFlags = [
+      "--enable-unicode=ucs4"
+      "--with-ensurepip=no"
+    ];
+  };
+
+  python3Wedge = mkPythonWedge {
+    version = "3.10.4";
+    hash = "sha256-gL+SX1cdpDazUhCIbPefbrX6XWxXExa3NWg0NFH3ehk=";
+    extraConfigureFlags = [
+      "--with-ensurepip=install"
+    ];
+  };
+
+  py3LibsCache = stdenv.mkDerivation {
+    name = "oils-py3-libs-cache";
+
+    nativeBuildInputs = [
+      python3Wedge
+      cacert
+    ];
+
+    dontUnpack = true;
+
+    buildPhase = ''
+      export HOME=$TMPDIR
+      export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
+
+      mkdir -p $out
+
+      ${python3Wedge}/bin/pip3 install --upgrade pip setuptools wheel
+
+      # Download exact versions (see requirements.txt)
+      ${python3Wedge}/bin/pip3 download -d $out --only-binary=:all: --no-deps \
+        -r ${./requirements.txt}
+    '';
+
+    dontInstall = true;
+
+    outputHashMode = "recursive";
+    outputHash = "sha256-dTO5D4vr42A7Pui0i8ggt8rD6xQplG17rGB76M4ntRY=";
+  };
+
+  mypyWedge = stdenv.mkDerivation (finalAttrs: {
+    pname = "oils-mypy-wedge";
+    version = "0.780";
+
+    src = fetchFromGitHub {
+      owner = "python";
+      repo = "mypy";
+      rev = "v${finalAttrs.version}";
+      hash = "sha256-czwCx6ZjCu3CrVmbI6NbstzWM0GvuPTWJiiUhXSznu4=";
+      fetchSubmodules = true;
+    };
+
+    nativeBuildInputs = [ python3Wedge ];
+
+    buildPhase = ''
+      runHook preBuild
+
+      export HOME=$TMPDIR
+
+      ${python3Wedge}/bin/python3 -m venv $out
+
+      # Install all packages from cached wheels (--no-deps to bypass version conflicts)
+      $out/bin/pip install \
+        --no-index \
+        --find-links=${py3LibsCache} \
+        --no-deps \
+        setuptools wheel packaging \
+        typing_extensions mypy_extensions typed_ast \
+        mypy
+
+      # Copy mypy source (mycpp extends mypy classes)
+      cp -r . $out/mypy-src
+
+      runHook postBuild
+    '';
+
+    dontInstall = true;
+    dontFixup = true;
+  });
+
+  readline-all = symlinkJoin {
+    name = "readline-all";
+    paths = [
+      readline
+      readline.dev
+    ];
+  };
+
+in
+{
+  inherit
+    python2Wedge
+    python3Wedge
+    mypyWedge
+    readline-all
+    ;
+}