0

pmap()在设置数组元素的值时，我在抛出 a时遇到了问题BoundsError——我的代码适用于 1 个工人，但不适用于 >1。我写了一个大致遵循真实代码流的最小工作示例：

  1. 获取源数据
  2. 定义要迭代的点集
  3. 初始化要计算的数组点
  4. 计算每个数组点

主文件：

#pmapdemo.jl
using Distributed
#addprocs(length(Sys.cpu_info())) # uncomment this line for error
@everywhere include(joinpath(@__DIR__, "pmapdemo2.jl"))

function main()
    # Get source data
    source = Dict{String, Any}("t"=>zeros(5),
                               "x"=>zeros(5,6),
                               "y"=>zeros(5,3),
                               "z"=>zeros(5,3))
    # Define set of points over which to iterate
    iterset = Dict{String, Any}("t"=>source["t"],
                                "x"=>source["x"],
                                "y"=>fill(2, size(source["t"])[1], 1),
                                "z"=>fill(2, size(source["t"])[1], 1))
    data = Dict{String, Any}()

    # Initialise array points to be calculated
    MyMod.initialisearray!(data, iterset)
    # Calculate each array point
    MyMod.calcarray!(data, iterset, source) 
    @show data
end

main()

功能文件：

#pmapdemo2.jl
module MyMod

using Distributed
@everywhere using SharedArrays

# Initialise data array 
function initialisearray!(data, fieldset)
    zerofield::SharedArray{Float64, 4} = zeros(size(fieldset["t"])[1],
                                               size(fieldset["x"])[2],
                                               size(fieldset["y"])[2],
                                               size(fieldset["z"])[2])
    data["field"] = deepcopy(zerofield)
end                            

# Calculate values of array elements according to values in source
function calcpoint!((data, source, a, b, c, d))
    data["field"][a,b,c,d] = rand()
end

# Set values in array
function calcarray!(data, iterset, source)
    for a in eachindex(iterset["t"])
        # [additional functionality f(a) here]
        b = eachindex(iterset["x"][a,:])
        c = eachindex(iterset["y"][a,:])
        d = eachindex(iterset["z"][a,:])
        pmap(calcpoint!, Iterators.product(Iterators.repeated(data,1), Iterators.repeated(source,1), Iterators.repeated(a,1), b, c, d))
    end
end

end

错误输出：

ERROR: LoadError: On worker 2:
BoundsError: attempt to access 0×0×0×0 Array{Float64,4} at index [1]
setindex! at ./array.jl:767 [inlined]
setindex! at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/SharedArrays/src/SharedArrays.jl:500 [inlined]
_setindex! at ./abstractarray.jl:1043
setindex! at ./abstractarray.jl:1020
calcpoint! at /home/dave/pmapdemo2.jl:25
#112 at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:269
run_work_thunk at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:56
macro expansion at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:269 [inlined]
#111 at ./task.jl:259
Stacktrace:
 [1] (::getfield(Base, Symbol("##696#698")))(::Task) at ./asyncmap.jl:178
 [2] foreach(::getfield(Base, Symbol("##696#698")), ::Array{Any,1}) at ./abstractarray.jl:1866
 [3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at ./asyncmap.jl:178
 [4] #async_usemap#681 at ./asyncmap.jl:154 [inlined]
 [5] #async_usemap at ./none:0 [inlined]
 [6] #asyncmap#680 at ./asyncmap.jl:81 [inlined]
 [7] #asyncmap at ./none:0 [inlined]
 [8] #pmap#213(::Bool, ::Int64, ::Nothing, ::Array{Any,1}, ::Nothing, ::Function, ::Function, ::WorkerPool, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:126
 [9] pmap(::Function, ::WorkerPool, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:101
 [10] #pmap#223(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Function, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:156
 [11] pmap(::Function, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:156
 [12] calcarray!(::Dict{String,Any}, ::Dict{String,Any}, ::Dict{String,Any}) at /home/dave/pmapdemo2.jl:20
 [13] main() at /home/dave/pmapdemo.jl:19
 [14] top-level scope at none:0
in expression starting at /home/dave/pmapdemo.jl:23

在pmapdemo2.jl中，替换data["field"][a,b,c,d] = rand()为@show a, b, c, d表明所有工作人员都在运行并且可以完全访问正在传递的变量，但是将其替换为@show data["field"]会引发相同的错误。SharedArrays 的全部目的肯定是为了避免这种情况吗？还是我误解了如何使用它pmap？

这是来自 Julia discourse here的交叉帖子。

4

1 回答 1

1

pmap将完成将数据传递给进程的工作，因此您不需要使用SharedArrays. 通常，提供给pmap（实际上是 map）的函数将是一个纯函数（因此不会改变任何变量），它返回输出数组的一个元素。该函数映射到输入数组的每个元素，该pmap函数将为您构造输出数组。例如，在您的情况下，代码可能看起来有点像这样

calcpoint(source, (a,b,c,d)) = rand() # Or some function of source and the indices a,b,c,d
field["data"] = pmap(calcpoint, Iterators.repeated(source), Iterators.product(a,b,c,d))
于 2019-08-28T14:54:26.363 回答