Pkg.jl: Trouble using newly-generated packages

Created on 21 Jun 2018  路  4Comments  路  Source: JuliaLang/Pkg.jl

I'm not exactly sure what fraction of this is supposed to work (julia Commit c1eb3e80e4* (2018-06-21 16:18 UTC)), but here's an annotated session where I had to hack access to one package and failed to figure out how to get a second package to make use of the first:

tim@diva:/tmp/pkgs$ julia-0.7 -q
(v0.7) pkg> init
Initialized project at /tmp/pkgs/Project.toml

(v0.7) pkg> generate A
Generating project A:
    A/Project.toml
    A/src/A.jl

julia> open("A/src/A.jl", "w") do io
           print(io, """
       __precompile__(true)

       module A
       f(x) = 1
       end
       """)
       end

julia> using A     # https://docs.julialang.org/en/latest/stdlib/Pkg/ seems to suggest this should work?
ERROR: ArgumentError: Package A not found in current path:
 - Run `Pkg.add("A")` to install the A package.

Stacktrace:
 [1] require(::Module, ::Symbol) at ./loading.jl:857

(v0.7) pkg> add A
   Cloning git-repo `/tmp/pkgs/A`
ERROR: GitError(Code:ENOTFOUND, Class:Repository, could not find repository from '/tmp/pkgs/A')
Stacktrace:
 [1] uv_error at ./libuv.jl:80 [inlined]
 [2] unlink(::String) at ./file.jl:715
 [3] #rm#9(::Bool, ::Bool, ::Function, ::String) at ./file.jl:253
 [4] #rm at ./none:0 [inlined]
 [5] #clone#2(::Nothing, ::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol},NamedTuple{(:isbare, :credentials),Tuple{Bool,LibGit2.CachedCredentials}}}, ::Function, ::String, ::String) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/GitTools.jl:81
 [6] #handle_repos_add!#16(::Bool, ::Function, ::Pkg.Types.Context, ::Array{Pkg.Types.PackageSpec,1}) at ./none:0
 [7] #handle_repos_add! at ./none:0 [inlined]
 [8] #add_or_develop#13(::Symbol, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Pkg.Types.Context, ::Array{Pkg.Types.PackageSpec,1}) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/API.jl:32
 [9] #add_or_develop at ./none:0 [inlined]
 [10] do_add_or_develop!(::Pkg.Types.Context, ::Array{Union{Pkg.Types.VersionRange, String, Pkg.REPLMode.Command, Pkg.REPLMode.Option, Pkg.REPLMode.Rev},1}, ::Pkg.REPLMode.CommandKind) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:592
 [11] #invokelatest#1 at ./essentials.jl:670 [inlined]
 [12] invokelatest at ./essentials.jl:669 [inlined]
 [13] do_cmd!(::Array{Union{Pkg.Types.VersionRange, String, Pkg.REPLMode.Command, Pkg.REPLMode.Option, Pkg.REPLMode.Rev},1}, ::REPL.LineEditREPL) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:262
 [14] #do_cmd#8(::Bool, ::Function, ::REPL.LineEditREPL, ::String) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:222
 [15] do_cmd at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:219 [inlined]
 [16] (::getfield(Pkg.REPLMode, Symbol("##27#30")){REPL.LineEditREPL,REPL.LineEdit.Prompt})(::REPL.LineEdit.MIState, ::Base.GenericIOBuffer{Array{UInt8,1}}, ::Bool) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:955
 [17] top-level scope at none:0

shell> ls
A  Project.toml

shell> cat Project.toml

julia> push!(LOAD_PATH, ".")    # this shouldn't be necessary, right?
3-element Array{String,1}:
 "@v#.#"  
 "@stdlib"
 "."      

julia> using A
ERROR: ArgumentError: Package A not found in current path:
 - Run `Pkg.add("A")` to install the A package.

Stacktrace:
 [1] require(::Module, ::Symbol) at ./loading.jl:857

shell> cd A
/tmp/pkgs/A

julia> using A     # yay!
[ Info: Precompiling module A

shell> cd ..
/tmp/pkgs

shell> cat Project.toml

(v0.7) pkg> add A
   Cloning git-repo `/tmp/pkgs/A`
ERROR: GitError(Code:ENOTFOUND, Class:Repository, could not find repository from '/tmp/pkgs/A')
Stacktrace:
 [1] uv_error at ./libuv.jl:80 [inlined]
 [2] unlink(::String) at ./file.jl:715
 [3] #rm#9(::Bool, ::Bool, ::Function, ::String) at ./file.jl:253
 [4] #rm at ./none:0 [inlined]
 [5] #clone#2(::Nothing, ::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol},NamedTuple{(:isbare, :credentials),Tuple{Bool,LibGit2.CachedCredentials}}}, ::Function, ::String, ::String) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/GitTools.jl:81
 [6] #handle_repos_add!#16(::Bool, ::Function, ::Pkg.Types.Context, ::Array{Pkg.Types.PackageSpec,1}) at ./none:0
 [7] #handle_repos_add! at ./none:0 [inlined]
 [8] #add_or_develop#13(::Symbol, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Pkg.Types.Context, ::Array{Pkg.Types.PackageSpec,1}) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/API.jl:32
 [9] #add_or_develop at ./none:0 [inlined]
 [10] do_add_or_develop!(::Pkg.Types.Context, ::Array{Union{Pkg.Types.VersionRange, String, Pkg.REPLMode.Command, Pkg.REPLMode.Option, Pkg.REPLMode.Rev},1}, ::Pkg.REPLMode.CommandKind) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:592
 [11] #invokelatest#1 at ./essentials.jl:670 [inlined]
 [12] invokelatest at ./essentials.jl:669 [inlined]
 [13] do_cmd!(::Array{Union{Pkg.Types.VersionRange, String, Pkg.REPLMode.Command, Pkg.REPLMode.Option, Pkg.REPLMode.Rev},1}, ::REPL.LineEditREPL) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:262
 [14] #do_cmd#8(::Bool, ::Function, ::REPL.LineEditREPL, ::String) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:222
 [15] do_cmd at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:219 [inlined]
 [16] (::getfield(Pkg.REPLMode, Symbol("##27#30")){REPL.LineEditREPL,REPL.LineEdit.Prompt})(::REPL.LineEdit.MIState, ::Base.GenericIOBuffer{Array{UInt8,1}}, ::Bool) at /home/tim/src/julia-1.0/usr/share/julia/stdlib/v0.7/Pkg/src/REPLMode.jl:955
 [17] top-level scope at none:0

(v0.7) pkg> generate B   # OK, now create B which will rely on A
Generating project B:
    B/Project.toml
    B/src/B.jl

julia> open("B/src/B.jl", "w") do io
           print(io, """
       __precompile__(true)

       module B
       using A
       g(x) = -1
       end
       """)
       end

julia> using B
ERROR: ArgumentError: Package B not found in current path:
 - Run `Pkg.add("B")` to install the B package.

Stacktrace:
 [1] require(::Module, ::Symbol) at ./loading.jl:857

shell> cd B
/tmp/pkgs/B

julia> using B
[ Info: Precompiling module B
ERROR: LoadError: ArgumentError: Package B does not have A in its dependencies:
 - If you have B checked out for development and have
   added A as a dependency but haven't updated your primary
   environment's manifest file, try `Pkg.resolve()`.
 - Otherwise you may need to report an issue with B.

Stacktrace:
 [1] require(::Module, ::Symbol) at ./loading.jl:862
 [2] include at ./boot.jl:317 [inlined]
 [3] include_relative(::Module, ::String) at ./loading.jl:1075
 [4] include(::Module, ::String) at ./sysimg.jl:29
 [5] top-level scope at none:0
 [6] eval at ./boot.jl:319 [inlined]
 [7] eval(::Expr) at ./client.jl:394
 [8] top-level scope at ./none:3 [inlined]
 [9] top-level scope at ./<missing>:0
in expression starting at /tmp/pkgs/B/src/B.jl:4
ERROR: Failed to precompile B to /home/tim/.julia/compiled/v0.7/B/ExSV.ji.
Stacktrace:
 [1] error at ./error.jl:33 [inlined]
 [2] compilecache(::Base.PkgId) at ./loading.jl:1205
 [3] _require(::Base.PkgId) at ./loading.jl:1007
 [4] require(::Base.PkgId) at ./loading.jl:879
 [5] require(::Module, ::Symbol) at ./loading.jl:874

(v0.7) pkg> add A
  Updating registry at `~/.julia/registries/Uncurated`
  Updating git-repo `https://github.com/JuliaRegistries/Uncurated.git`
ERROR: The following package names could not be resolved:
 * A (not found in project, manifest or registry)
Please specify by known `name=uuid`.
picture timholy

Most helpful comment

Is there any way to add it during generate, though?

Yes, perhaps it makes sense to add the generated project automatically?

I think the example in your last post worked pre-https://github.com/JuliaLang/julia/pull/27633 where the active environment was automatically determined by pwd(). The problem now is that after

shell> cd B
/tmp/pkgs/

we don't activate Bs environment, as indicated by the prompt which is still (v0.7) pkg>. If we had activated Bs environment the prompt would be (B) pkg>. This means that in the next step, the add A is a no-op -- A is already added to the (v0.7) environment. To activate another environment you can, for now, export JULIA_LOAD_PATH="@:" which will be expanded to the directory where you start julia, and with https://github.com/JuliaLang/julia/pull/27696 we can set it to "@@:" to essentially get the pre-https://github.com/JuliaLang/julia/pull/27633 behaviour back. Ideally there would be a pkg> activate or similar command to easily switch environments within the same julia process, which would make things like this a lot smoother.

Here is how you can make this work now (with export JULIA_LOAD_PATH="@:")

(v0.7) pkg> generate A
Generating project A:
    A/Project.toml
    A/src/A.jl

julia> open("A/src/A.jl", "w") do io
           print(io, """
       __precompile__(true)

       module A
       f(x) = 1
       end
       """)
       end

(v0.7) pkg> dev A
 Resolving package versions...
  Updating `Project.toml`
  [f254ec9a] + A v0.1.0 [`A`]
  Updating `Manifest.toml`
  [f254ec9a] + A v0.1.0 [`A`]

(v0.7) pkg> generate B
Generating project B:
    B/Project.toml
    B/src/B.jl

julia> open("B/src/B.jl", "w") do io
           print(io, """
       __precompile__(true)

       module B
       using A
       g(x) = -1
       end
       """)
       end

(v0.7) pkg> dev B
 Resolving package versions...
  Updating `Project.toml`
  [0fdf17cc] + B v0.1.0 [`B`]
  Updating `Manifest.toml`
  [0fdf17cc] + B v0.1.0 [`B`]

cd to B and start julia in this directory (note from the prompt that the active environment is B)

(B) pkg> dev ../A
 Resolving package versions...
  Updating `Project.toml`
  [f254ec9a] + A v0.1.0 [`~/A`]
  Updating `Manifest.toml`
  [f254ec9a] + A v0.1.0 [`~/A`]

Now we added A to Bs Project.toml (done with dev though, since we want to point it to a specific local path in this case). Now cd back and start julia again

(v0.7) pkg> st
    Status `Project.toml`
  [f254ec9a] A v0.1.0 [`A`]
  [0fdf17cc] B v0.1.0 [`B`]

julia> using A
[ Info: Precompiling module A

julia> using B
[ Info: Precompiling module B

julia> A.f(2)
1

julia> B.g(3)
-1
picture fredrikekre on 22 Jun 2018
❤2

All 4 comments

Seems like the dev HelloWorld step, to add it to the project, is missing from the docs. Should be something like this I think

(v0.7) pkg> generate HelloWorld
Generating project HelloWorld:
    HelloWorld/Project.toml
    HelloWorld/src/HelloWorld.jl

(v0.7) pkg> dev HelloWorld
 Resolving package versions...
  Updating `Project.toml`
  [7c175e58] + HelloWorld v0.1.0 [`HelloWorld`]
  Updating `Manifest.toml`
  [7c175e58] + HelloWorld v0.1.0 [`HelloWorld`]

julia> import HelloWorld

julia> HelloWorld.greet()
Hello World!
fredrikekre picture fredrikekre on 22 Jun 2018
👍1

Thanks, that helps. I can try to edit the docs. Is there any way to add it during generate, though? I think anyone who just fires up an editor and started writing code may be surprised. (But I should say I am not sure I fully understand the purpose of dev, maybe because I'm always deving :grin: .)

What about dependencies, though?

(v0.7) pkg> init
Initialized project at /tmp/pkgs/Project.toml

(v0.7) pkg> generate A
Generating project A:
    A/Project.toml
    A/src/A.jl

julia> open("A/src/A.jl", "w") do io
           print(io, """
       __precompile__(true)

       module A
       f(x) = 1
       end
       """)
       end

(v0.7) pkg> dev A
 Resolving package versions...
  Updating `~/.julia/environments/v0.7/Project.toml`
  [b2fc4464] + A v0.1.0 [`/tmp/pkgs/A`]
  Updating `~/.julia/environments/v0.7/Manifest.toml`
  [b2fc4464] + A v0.1.0 [`/tmp/pkgs/A`]

(v0.7) pkg> generate B
Generating project B:
    B/Project.toml
    B/src/B.jl

julia> open("B/src/B.jl", "w") do io
           print(io, """
       __precompile__(true)

       module B
       using A
       g(x) = -1
       end
       """)
       end

(v0.7) pkg> dev B
 Resolving package versions...
  Updating `~/.julia/environments/v0.7/Project.toml`
  [0759b618] + B v0.1.0 [`/tmp/pkgs/B`]
  Updating `~/.julia/environments/v0.7/Manifest.toml`
  [0759b618] + B v0.1.0 [`/tmp/pkgs/B`]

shell> ls
A  B  Project.toml

shell> cd B
/tmp/pkgs/B

(v0.7) pkg> add A
  Updating registry at `~/.julia/registries/Uncurated`
  Updating git-repo `https://github.com/JuliaRegistries/Uncurated.git`
 Resolving package versions...
  Updating `~/.julia/environments/v0.7/Project.toml`
 [no changes]
  Updating `~/.julia/environments/v0.7/Manifest.toml`
 [no changes]

(v0.7) pkg> resolve
 Resolving package versions...
  Updating `~/.julia/environments/v0.7/Project.toml`
 [no changes]
  Updating `~/.julia/environments/v0.7/Manifest.toml`
 [no changes]

shell> cat Project.toml
authors = ["Tim Holy <[email protected]>"]
name = "B"
uuid = "0759b618-75b2-11e8-12bf-7971e41ce912"
version = "0.1.0"

[deps]

shell> cd ..
/tmp/pkgs

julia> using A
[ Info: Precompiling module A

julia> using B
[ Info: Precompiling module B
ERROR: LoadError: ArgumentError: Package B does not have A in its dependencies:
 - If you have B checked out for development and have
   added A as a dependency but haven't updated your primary
   environment's manifest file, try `Pkg.resolve()`.
 - Otherwise you may need to report an issue with B.

Stacktrace:
 [1] require(::Module, ::Symbol) at ./loading.jl:862
 [2] include at ./boot.jl:317 [inlined]
 [3] include_relative(::Module, ::String) at ./loading.jl:1075
 [4] include(::Module, ::String) at ./sysimg.jl:29
 [5] top-level scope at none:0
 [6] eval at ./boot.jl:319 [inlined]
 [7] eval(::Expr) at ./client.jl:394
 [8] top-level scope at ./none:3 [inlined]
 [9] top-level scope at ./<missing>:0
in expression starting at /tmp/pkgs/B/src/B.jl:4
ERROR: Failed to precompile B to /home/tim/.julia/compiled/v0.7/B/5WMh.ji.
Stacktrace:
 [1] error at ./error.jl:33 [inlined]
 [2] compilecache(::Base.PkgId) at ./loading.jl:1205
 [3] _require(::Base.PkgId) at ./loading.jl:1007
 [4] require(::Base.PkgId) at ./loading.jl:879
 [5] require(::Module, ::Symbol) at ./loading.jl:874
timholy picture timholy on 22 Jun 2018

Is there any way to add it during generate, though?

Yes, perhaps it makes sense to add the generated project automatically?

I think the example in your last post worked pre-https://github.com/JuliaLang/julia/pull/27633 where the active environment was automatically determined by pwd(). The problem now is that after

shell> cd B
/tmp/pkgs/

we don't activate Bs environment, as indicated by the prompt which is still (v0.7) pkg>. If we had activated Bs environment the prompt would be (B) pkg>. This means that in the next step, the add A is a no-op -- A is already added to the (v0.7) environment. To activate another environment you can, for now, export JULIA_LOAD_PATH="@:" which will be expanded to the directory where you start julia, and with https://github.com/JuliaLang/julia/pull/27696 we can set it to "@@:" to essentially get the pre-https://github.com/JuliaLang/julia/pull/27633 behaviour back. Ideally there would be a pkg> activate or similar command to easily switch environments within the same julia process, which would make things like this a lot smoother.

Here is how you can make this work now (with export JULIA_LOAD_PATH="@:")

(v0.7) pkg> generate A
Generating project A:
    A/Project.toml
    A/src/A.jl

julia> open("A/src/A.jl", "w") do io
           print(io, """
       __precompile__(true)

       module A
       f(x) = 1
       end
       """)
       end

(v0.7) pkg> dev A
 Resolving package versions...
  Updating `Project.toml`
  [f254ec9a] + A v0.1.0 [`A`]
  Updating `Manifest.toml`
  [f254ec9a] + A v0.1.0 [`A`]

(v0.7) pkg> generate B
Generating project B:
    B/Project.toml
    B/src/B.jl

julia> open("B/src/B.jl", "w") do io
           print(io, """
       __precompile__(true)

       module B
       using A
       g(x) = -1
       end
       """)
       end

(v0.7) pkg> dev B
 Resolving package versions...
  Updating `Project.toml`
  [0fdf17cc] + B v0.1.0 [`B`]
  Updating `Manifest.toml`
  [0fdf17cc] + B v0.1.0 [`B`]

cd to B and start julia in this directory (note from the prompt that the active environment is B)

(B) pkg> dev ../A
 Resolving package versions...
  Updating `Project.toml`
  [f254ec9a] + A v0.1.0 [`~/A`]
  Updating `Manifest.toml`
  [f254ec9a] + A v0.1.0 [`~/A`]

Now we added A to Bs Project.toml (done with dev though, since we want to point it to a specific local path in this case). Now cd back and start julia again

(v0.7) pkg> st
    Status `Project.toml`
  [f254ec9a] A v0.1.0 [`A`]
  [0fdf17cc] B v0.1.0 [`B`]

julia> using A
[ Info: Precompiling module A

julia> using B
[ Info: Precompiling module B

julia> A.f(2)
1

julia> B.g(3)
-1
fredrikekre picture fredrikekre on 22 Jun 2018
❤2

Seems like there is nothing left to do here

00vareladavid picture 00vareladavid on 9 Feb 2019
👍1
Was this page helpful?
0 / 5 - 0 ratings

Related issues

KristofferC picture KristofferC  路  4Comments
cossio picture cossio  路  3Comments
cscherrer picture cscherrer  路  3Comments
MaWo2 picture MaWo2  路  4Comments
GordStephen picture GordStephen  路  3Comments