Nixpkgs: Can't build Arpack in Julia

[suhr]

julia> using LsqFit
[ Info: Precompiling LsqFit [2fda8390-95c7-5789-9bda-21331edee243]
ERROR: LoadError: No deps.jl file could be found. Please try running Pkg.build("Arpack").
Currently, the build command might fail when Julia has been built from source
and the recommendation is to use the official binaries from julialang.org.
For more info see https://github.com/JuliaLinearAlgebra/Arpack.jl/issues/5.

julia> Pkg.build("Arpack")
  Building Arpack → `~/.julia/packages/Arpack/cu5By/deps/build.log`
┌ Error: Error building `Arpack`:
│ ERROR: LoadError: LibraryProduct(nothing, ["libarpack"], :libarpack, "Prefix(/home/suhr/.julia/packages/Arpack/cu5By/deps/usr)") is not satisfied, cannot generate deps.jl!
│ Stacktrace:
│  [1] error(::String) at /nix/store/kh9x1nnrxmkk231nkpp952pqjknm7pnh-julia-1.0.4/lib/julia/sys.so:?
│  [2] #write_deps_file#152(::Bool, ::Function, ::String, ::Array{LibraryProduct,1}) at /home/suhr/.julia/packages/BinaryProvider/TcAwt/src/Products.jl:414
│  [3] (::getfield(BinaryProvider, Symbol("#kw##write_deps_file")))(::NamedTuple{(:verbose,),Tuple{Bool}}, ::typeof(write_deps_file), ::String, ::Array{LibraryProduct,1}) at ./none:0
│  [4] top-level scope at none:0
│  [5] include_relative(::Module, ::String) at /nix/store/kh9x1nnrxmkk231nkpp952pqjknm7pnh-julia-1.0.4/lib/julia/sys.so:?
│  [6] include(::Module, ::String) at /nix/store/kh9x1nnrxmkk231nkpp952pqjknm7pnh-julia-1.0.4/lib/julia/sys.so:?
│  [7] include(::String) at ./client.jl:392
│  [8] top-level scope at none:0
│ in expression starting at /home/suhr/.julia/packages/Arpack/cu5By/deps/build.jl:74
└ @ Pkg.Operations /build/source/usr/share/julia/stdlib/v1.0/Pkg/src/Operations.jl:1096

nix run nixpkgs.nix-info -c nix-info -m output:

 - system: `"x86_64-linux"`
 - host os: `Linux 4.19.71, NixOS, 19.09pre192418.e19054ab3cd (Loris)`
 - multi-user?: `yes`
 - sandbox: `yes`
 - version: `nix-env (Nix) 2.3`
 - channels(root): `"home-manager, nixos-19.09pre192418.e19054ab3cd"`
 - nixpkgs: `/nix/var/nix/profiles/per-user/root/channels/nixos`

[veprbl]

I believe @tbenst had some success with it in https://github.com/NixOS/nixpkgs/issues/54634#issuecomment-481785619

[tbenst]

Yes I did indeed!

[suhr]

Having to use FHS is really unfortunate. I hope there's a better way.

[tbenst]

@suhr here's a non-FHS solution:

{pkgs, stdenv, ...}:

with pkgs;
let
  julia = julia_11;
  d = version: "v${lib.concatStringsSep "." (lib.take 2 (lib.splitString "." version))}";
  extraLibs = [
    # IJulia.jl
    mbedtls
    zeromq3
    # ImageMagick.jl
    imagemagickBig
    # HDF5.jl
    hdf5
    # Cairo.jl
    cairo
    gettext
    pango.out
    glib.out
    # Gtk.jl
    gtk3
    gdk_pixbuf
    # GZip.jl # Required by DataFrames.jl
    gzip
    zlib
    # GR.jl # Runs even without Xrender and Xext, but cannot save files, so those are required
    xorg.libXt
    xorg.libX11
    xorg.libXrender
    xorg.libXext
    glfw
    freetype
    # Flux.jl
    cudatoolkit
    linuxPackages.nvidia_x11
    git gitRepo gnupg autoconf curl
    procps gnumake utillinux m4 gperf unzip
    libGLU_combined
    xorg.libXi xorg.libXmu freeglut
    xorg.libXext xorg.libX11 xorg.libXv xorg.libXrandr zlib
    ncurses5 stdenv.cc binutils
    # Arpack.jl
    arpack
    gfortran.cc
    (pkgs.runCommand "openblas64_" {} ''
      mkdir -p "$out"/lib/
      ln -s ${openblasCompat}/lib/libopenblas.so "$out"/lib/libopenblas64_.so.0
    '')
  ];
in
stdenv.mkDerivation rec {
  name = "julia-env";
  version = julia.version;
  nativeBuildInputs = [ makeWrapper cacert git pkgconfig which ];
  buildInputs = [
    julia
    /* jupyterEnv  # my custom jupyter */
  ] ++ extraLibs;
  phases = [ "installPhase" ];
  installPhase = ''
    export CUDA_PATH="${cudatoolkit}"
    export LD_LIBRARY_PATH=${lib.makeLibraryPath extraLibs}
    # pushd $JULIA_PKGDIR/${d version}
    makeWrapper ${julia}/bin/julia $out/bin/julia \
        --prefix LD_LIBRARY_PATH : "$LD_LIBRARY_PATH" \
        --prefix LD_LIBRARY_PATH ":" "${linuxPackages.nvidia_x11}/lib" \
        --set CUDA_PATH "${cudatoolkit}" \
        --set JULIA_PKGDIR $JULIA_PKGDIR
        # --set JULIA_LOAD_PATH $JULIA_PKGDIR/${d version}
  '';
}

I have this saved to /Computer/packages/julia.nix, and also have the following saved to /home/tyler/.config/nixpkgs/overlays.nix:

[
(self: super: {
  cudatoolkit = super.cudatoolkit_10_0;
  cudnn_cudatoolkit = super.cudnn_cudatoolkit_10_0;
  julia = super.callPackage /Computer/packages/julia.nix { };
})
]

Hope this helps, and appreciate any suggested improvements!

[karetsu]

I don't need all of the other stuff, I just want to get Arpack installed so I can use Gadfly, so I wrote the following default.nix:

with import <nixpkgs> {};

stdenv.mkDerivation rec {
  name = "julia-env";
  version = julia_11.version;
  buildInputs = [
    julia_11
    arpack
    gfortran.cc
    (pkgs.runCommand "openblas64_" {} ''
      mkdir -p "$out"/lib/
      ln -s ${openblasCompat}/lib/libopenblas.so "$out"/lib/libopenblas64_.so.0
    '')
  ];
}

and using ]add Arpack in the julia within my nix-shell:

[nix-shell:~/Projects/documentation/synthetic]$ type julia
julia is /nix/store/rf86v5scp515l92gb5fff6rsjy25r2i1-julia-1.1.1/bin/julia

still yields me:

  Building Arpack → `~/.julia/packages/Arpack/zCmTA/deps/build.log`
┌ Error: Error building `Arpack`: 
│ [ Info: Downloading https://github.com/JuliaBinaryWrappers/Arpack_jll.jl/releases/download/Arpack-v3.5.0+2/Arpack.v3.5.0.x86_64-linux-gnu-libgfortran4.tar.gz to /home/aloysius/.julia/packages/Arpack/zCmTA/deps/usr/downloads/Arpack.v3.5.0.x86_64-linux-gnu-libgfortran4.tar.gz...
│ ERROR: LoadError: LibraryProduct(nothing, ["libarpack"], :libarpack, "Prefix(/home/aloysius/.julia/packages/Arpack/zCmTA/deps/usr)") is not satisfied, cannot generate deps.jl!
│ Stacktrace:
│  [1] error(::String) at /nix/store/rf86v5scp515l92gb5fff6rsjy25r2i1-julia-1.1.1/lib/julia/sys.so:?
│  [2] #write_deps_file#152(::Bool, ::Function, ::String, ::Array{LibraryProduct,1}) at /home/aloysius/.julia/packages/BinaryProvider/kcGxO/src/Products.jl:414
│  [3] (::getfield(BinaryProvider, Symbol("#kw##write_deps_file")))(::NamedTuple{(:verbose,),Tuple{Bool}}, ::typeof(write_deps_file), ::String, ::Array{LibraryProduct,1}) at ./none:0
│  [4] top-level scope at none:0
│  [5] include_relative(::Module, ::String) at /nix/store/rf86v5scp515l92gb5fff6rsjy25r2i1-julia-1.1.1/lib/julia/sys.so:?
│  [6] include(::Module, ::String) at /nix/store/rf86v5scp515l92gb5fff6rsjy25r2i1-julia-1.1.1/lib/julia/sys.so:?
│  [7] include(::String) at ./client.jl:403
│  [8] top-level scope at none:0
│ in expression starting at /home/aloysius/.julia/packages/Arpack/zCmTA/deps/build.jl:74
[19:03:41] ######################################################################## 100.0% 
└ @ Pkg.Operations /build/source/usr/share/julia/stdlib/v1.1/Pkg/src/Operations.jl:1075

On inspection it has downloaded Arpack in deps/usr/download and the sha256 sums match so is this more broken than before?
nix run nixpkgs.nix-info -c nix-info -m output:

 - system: `"x86_64-linux"`
 - host os: `Linux 4.19.92, NixOS, 19.09.1748.ad1e1af5ad3 (Loris)`
 - multi-user?: `yes`
 - sandbox: `yes`
 - version: `nix-env (Nix) 2.3.1`
 - channels(aloysius): `"home-manager-19.09"`
 - channels(root): `"nixos-19.09.1748.ad1e1af5ad3"`
 - nixpkgs: `/nix/var/nix/profiles/per-user/root/channels/nixos`

[tbenst]

@karetsu you still need to use makeWrapper on julia or else it won't find libarpack. Specifically,

makeWrapper ${julia}/bin/julia $out/bin/julia \
        --prefix LD_LIBRARY_PATH : "$LD_LIBRARY_PATH"

I don't remember why --prefix JULIA_PKGDIR $JULIA_PKGDIR was there but try that too as needed

[karetsu]

@tbenst my bad, I was using nix-shell rather than nix-build. Building it then dropping into the shell works fine. Consider the previous comment redacted.

[tbenst]

@karetsu also note that you may want to specify julia_11 and not wrap julia (1.0). Did you get it working? if so please post for others, thanks!

[karetsu]

@tbenst yes, it even worked with julia (not just julia_11) and I'm now happily using gadfly. I did notice that after the nix-build type julia still was odd so the steps I went through were the following:

1. created default.nix with the content posted below
2. run nix-build in the directory it was created in
3. run /nix/store/*julia-env/bin/julia
4. ]add Gadfly
5. using Gadfly (make sure everything is all compiled and nice)

After this I did not need to use julia-env which was in /nix/store/ but ofc should I want to change minor version of julia at some point I would need to use it again to recompile Arpack.jl

default.nix

with import <nixpkgs> { };

let
  extraLibs = [
    arpack
    gfortran.cc
    (pkgs.runCommand "openblas64_" { } ''
      mkdir -p "$out"/lib/
      ln -s ${openblasCompat}/lib/libopenblas.so "$out"/lib/libopenblas64_.so.0
    '')
  ];
in stdenv.mkDerivation rec {
  name = "julia-env";
  version = julia.version;
  nativeBuildInputs = [ makeWrapper cacert git pkgconfig which ];
  buildInputs = [ julia_11 ] ++ extraLibs;
  phases = "installPhase";
  installPhase = ''
    export LD_LIBRARY_PATH=${lib.makeLibraryPath extraLibs}
    makeWrapper ${julia}/bin/julia $out/bin/julia \
        --prefix LD_LIBRARY_PATH : "$LD_LIBRARY_PATH" \
        --set JULIA_PKGDIR $JULIA_PKGDIR
  '';
}

I'm certain this is not the idiomatic nix way to do it but all I needed was a compiled Arpack and then everything else is suitable within my standard julia install.

[tbenst]

hmm you really want the two versions to match, or else you may have undefined behavior where julia 1.0 and 1.1 differ:

with import <nixpkgs> { };

let
  extraLibs = [
    arpack
    gfortran.cc
    (pkgs.runCommand "openblas64_" { } ''
      mkdir -p "$out"/lib/
      ln -s ${openblasCompat}/lib/libopenblas.so "$out"/lib/libopenblas64_.so.0
    '')
  ];
in stdenv.mkDerivation rec {
  name = "julia-env";
  version = julia.version;
  nativeBuildInputs = [ makeWrapper cacert git pkgconfig which ];
  buildInputs = [ julia_11 ] ++ extraLibs;
  phases = "installPhase";
  installPhase = ''
    export LD_LIBRARY_PATH=${lib.makeLibraryPath extraLibs}
    makeWrapper ${julia_11}/bin/julia $out/bin/julia \
        --prefix LD_LIBRARY_PATH : "$LD_LIBRARY_PATH" \
        --set JULIA_PKGDIR $JULIA_PKGDIR
  '';
}

[karetsu]

ah yes, that was a typo sorry (tried changing it back to _11 for posting) but I would think that those who would do it properly will use this as their daily driver version of julia so yours should be the one used.

[stale[bot]]

Thank you for your contributions.

This has been automatically marked as stale because it has had no activity for 180 days.

If this is still important to you, we ask that you leave a comment below. Your comment can be as simple as "still important to me". This lets people see that at least one person still cares about this. Someone will have to do this at most twice a year if there is no other activity.

Here are suggestions that might help resolve this more quickly:

1. Search for maintainers and people that previously touched the related code and @ mention them in a comment.
2. Ask on the NixOS Discourse.
3. Ask on the #nixos channel on irc.freenode.net.

[nixos-discourse]

This issue has been mentioned on NixOS Discourse. There might be relevant details there:

https://discourse.nixos.org/t/how-can-we-build-a-julia-derivation-with-tests-disabled/8052/1