smillerc / mpihaloarrays.jl Goto Github PK
View Code? Open in Web Editor NEWAn array type for MPI halo data exchange in Julia
License: MIT License
An array type for MPI halo data exchange in Julia
License: MIT License
this is my code:
using MPI, MPIHaloArrays
using FortranFiles
using CircularArrays
using JLD2
using Statistics
MPI.Init()
const comm = MPI.COMM_WORLD
const rank = MPI.Comm_rank(comm)
const nprocs = MPI.Comm_size(comm)
const root = 0 # root rank
@assert nprocs == 40 "This example is designed with $(nprocs) processes..."
function gauss_filter_parallel_1d(data::AbstractArray{T, 3}, radius, nhalo::Int) where T<:Real
nx, ny, nz = size(data)
filtered_data = zeros(eltype(data),nx,ny,nz)
data = CircularArray(data[:,:,:])
dz = 14.0 / nz
r_gridz = ceil(Int, radius / dz)
G = zeros(2*r_gridz+1)
for k in 1:2*r_gridz+1
dis_z = (k-r_gridz-1)*dz
G[k] = sqrt(6/(π*4*radius^2))*exp(-6*(dis_z)^2/(4*radius^2))
end
G = G .* size(G) ./ sum(G)
for i in 1+nhalo:nx+nhalo
for j in 1+nhalo:ny+nhalo
for k in 1+nhalo:nz+nhalo
kmin = k-r_gridz
kmax = k+r_gridz
filtered_data[i,j,k] = mean( data[i, j, kmin:kmax] .* G)
end
end
end
return filtered_data
end
function main(filepath, nx_g, ny_g, nz_g, nx, ny, nz, nhalo,
topology, steps, Δ, δᵥ, Ndims)
for n in steps
f = "$(filepath)OCFD$(string(n,pad=7)).dat"
flow_data = rand(nx, ny, nz, 5)#initializer3D(f, nx_g, ny_g, nz_g, Ndims, root, nhalo, topology)
ρ = MPIHaloArray(flow_data[:,:,:,1], topology, nhalo; do_corners = false)
u = MPIHaloArray(flow_data[:,:,:,2], topology, nhalo; do_corners = false)
v = MPIHaloArray(flow_data[:,:,:,3], topology, nhalo; do_corners = false)
w = MPIHaloArray(flow_data[:,:,:,4], topology, nhalo; do_corners = false)
T = MPIHaloArray(flow_data[:,:,:,5], topology, nhalo; do_corners = false)
fillhalo!(ρ, 0)
fillhalo!(u, 0)
fillhalo!(v, 0)
fillhalo!(w, 0)
fillhalo!(T, 0)
updatehalo!(ρ)
updatehalo!(u)
updatehalo!(v)
updatehalo!(w)
updatehalo!(T)
ρ1 = ρ.data
u1 = u.data .* ρ.data
v1 = v.data .* ρ.data
w1 = w.data .* ρ.data
T1 = T.data .* ρ.data
for m in Δ
gl = δᵥ * m / 2
ρ.data = gauss_filter_parallel_1d(ρ1, gl, nhalo)
u.data = gauss_filter_parallel_1d(u1, gl, nhalo)
v.data = gauss_filter_parallel_1d(v1, gl, nhalo)
w.data = gauss_filter_parallel_1d(w1, gl, nhalo)
T.data = gauss_filter_parallel_1d(T1, gl, nhalo)
G1=gatherglobal(ρ)
G2=gatherglobal(u)
G3=gatherglobal(v)
G4=gatherglobal(w)
G5=gatherglobal(T)
if rank == root
G2 ./= G1
G3 ./= G1
G4 ./= G1
G5 ./= G1
println("Test$(string(n,pad=7))_g_filt_$(m).jld2")
end # end rank == 0
end #end Δ loop
end #end for n loop
end
Ndims = 5
δᵥ = 0.0355
npx0, npy0, npz0 = 1, 40, 1
nx_g, ny_g, nz_g = 1401, 160, 128
halo_dims = (1, 2, 3)
sizes = MPIHaloArrays.get_subdomain_sizes((nx_g,ny_g,nz_g),(npx0,npy0,npz0),halo_dims)
nx, ny, nz = sizes[:,rank + 1] |> Tuple
nhalo = 0
topology = CartesianTopology(comm, [npx0,npy0,npz0], [false, false, true])
filepath = "/home��u_jyc/Data/M29/Flateplate/Tw284/70W-80W/"
steps = range(start = 798100, step = 100, stop = 798500)
Δ = [15, 60, 120, 240]# .* δᵥ ./ 2.0
main(filepath, nx_g, ny_g, nz_g, nx, ny, nz, nhalo, topology, steps, Δ, δᵥ, Ndims)
GC.gc()
MPI.Finalize()
this is the memory allocated, if allocated memory beyond my cp, it will be killed.
This issue is used to trigger TagBot; feel free to unsubscribe.
If you haven't already, you should update your TagBot.yml
to include issue comment triggers.
Please see this post on Discourse for instructions and more details.
If you'd like for me to do this for you, comment TagBot fix
on this issue.
I'll open a PR within a few hours, please be patient!
I was trying to use the package with Julia v1.9.1.
Installed MPIHaloArrays ──── v0.4.0
Updating `<PATH>\Project.toml`
[e77e6532] + MPIHaloArrays v0.4.0
Updating `<PATH>\Manifest.toml`
[79e6a3ab] + Adapt v3.6.2
[4fba245c] + ArrayInterface v7.4.8
[34da2185] + Compat v4.6.1
[da5c29d0] + EllipsisNotation v1.7.0
[615f187c] + IfElse v0.1.1
[e77e6532] + MPIHaloArrays v0.4.0
[6fe1bfb0] + OffsetArrays v1.12.9
[66db9d55] + SnoopPrecompile v1.0.3
[aedffcd0] + Static v0.8.7
[0d7ed370] + StaticArrayInterface v1.4.0
[4607b0f0] + SuiteSparse
Building MPIHaloArrays → `<PATH>\.julia\scratchspaces\44cfe95a-1eb2-52ea-b672-e2afdf69b78f\ca1144059b37b3e0792e5e8524328a65abd1d7ae\build.log`
ERROR: Error building `MPIHaloArrays`:
ERROR: LoadError: KeyError: key "HOME" not found
Stacktrace:
[1] (::Base.var"#693#694")(k::String)
@ Base .\env.jl:100
[2] access_env(onError::Base.var"#693#694", str::String)
@ Base .\env.jl:14
[3] getindex(#unused#::Base.EnvDict, k::String)
@ Base .\env.jl:100
[4] top-level scope
@ <PATH>\.julia\packages\MPIHaloArrays\pyqZ5\deps\build.jl:2
[5] include(fname::String)
@ Base.MainInclude .\client.jl:478
[6] top-level scope
@ none:5
in expression starting at <PATH>\.julia\packages\MPIHaloArrays\pyqZ5\deps\build.jl:2
The project/environement was fresh new, with minimal packages added. Project.toml content:
[da04e1cc] MPI v0.20.10
[e77e6532] MPIHaloArrays v0.4.0
[90137ffa] StaticArrays v1.5.26
In running the 2D and 3D examples from the documentation, update halo!
fails with MPI_ERR_RANK: invalid rank
when do_corners=true
for an MPIHaloArray. There is no error when do_corners=false
. I have tested this in both the current release, v0.2.0 and downloading the package from Github in Julia using Pkg> add MPIHaloArrays#a5e9dd2
.
Minimum Working Example (ran with mpiexecjl -np 8 julia mpi_test.jl
:
using MPI
using MPIHaloArrays
using CUDA
MPI.Init()
const comm = MPI.COMM_WORLD
const rank = MPI.Comm_rank(comm)
const nprocs = MPI.Comm_size(comm)
@assert nprocs == 8 "This example is designed with 8 processes..."
function print_array(U, proc)
if rank == proc
println("rank: ", proc)
display(U.data)
println()
end
MPI.Barrier(comm)
end
function print_topo(U,proc)
if rank == proc
println(U.topology)
end
MPI.Barrier(comm)
end
topo = CartesianTopology(comm,[4,2], # use a 4x2 decomposition
[false, false])
nhalo = 2
ni = 6
nj = 5
A = MPIHaloArray(zeros(Int, ni, nj), topo, nhalo,do_corners=true)
for p in 0:nprocs-1
print_topo(A,p)
end
fillhalo!(A, -1)
filldomain!(A, rank)
if rank == 0 println("Before sync") end
if rank == 0 println(A.do_corners) end
for p in 0:nprocs-1
print_array(A, p)
end
updatehalo!(A)
if rank == 0 println("After sync") end
for p in 0:nprocs-1
print_array(A, p)
end
GC.gc()
MPI.Finalize()
Error Output (program output removed)
ERROR: ERROR: ERROR: ERROR: ERROR: ERROR: ERROR: LoadError: ERROR: LoadError: LoadError: LoadError: LoadError: LoadError: LoadError: LoadError: MPIError(MPIError(MPIError(MPIError(6): MPI_ERR_RANK: invalid rank6): MPI_ERR_RANK: invalid rank6): MPI_ERR_RANK: invalid rank6): MPI_ERR_RANK: invalid rank
Stacktrace:
Stacktrace:
Stacktrace:
Stacktrace:
[1] [1] [1] [1] IsendIsend(Isend((buf::bufbuf::Isend::(buf::MPIError(MPIError(6): MPI_ERR_RANK: invalid rank6): MPI_ERR_RANK: invalid rank
Stacktrace:
Stacktrace:
[1] [1] Isend(bufIsend(buf::::MPIError(6): MPI_ERR_RANK: invalid rank
Stacktrace:
[1] Isend(buf::MPI.Buffer{Matrix{Int64}}, dest::Int64, tag::Int64, comm::MPI.Comm)
@ MPI ~/.julia/packages/MPI/08SPr/src/pointtopoint.jl:229
[2] updatehalo!(A::MPIHaloArray{Int64, 2, Matrix{Int64}, 2})
@ MPIHaloArrays ~/.julia/packages/MPIHaloArrays/WDydQ/src/updatehalo.jl:352
[3] top-level scope
@ ~/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
in expression starting at /home/matt/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
MPI.Buffer{Matrix{Int64}}, dest::Int64, tag::Int64, comm::MPI.Comm)
@ MPI ~/.julia/packages/MPI/08SPr/src/pointtopoint.jl:229
[2] updatehalo!(A::MPIHaloArray{Int64, 2, Matrix{Int64}, 2})
@ MPIHaloArrays ~/.julia/packages/MPIHaloArrays/WDydQ/src/updatehalo.jl:351
[3] top-level scope
@ ~/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
in expression starting at /home/matt/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
MPIError(MPI.Buffer{Matrix{Int64}}, dest::Int64, tag::Int64, comm::MPI.Comm)
@ MPI ~/.julia/packages/MPI/08SPr/src/pointtopoint.jl:229
[2] updatehalo!(A::MPI.Buffer{Matrix{Int64}}, dest::Int64, tag::Int64, comm::MPI.Comm)
@ MPI ~/.julia/packages/MPI/08SPr/src/pointtopoint.jl:229
[2] updatehalo!(A::MPIHaloArray{Int64, 2, Matrix{Int64}, 2})
@ MPIHaloArrays ~/.julia/packages/MPIHaloArrays/WDydQ/src/updatehalo.jl:352
[3] top-level scope
@ ~/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
in expression starting at /home/matt/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
MPIHaloArray{Int64, 2, Matrix{Int64}, 2})
@ MPIHaloArrays ~/.julia/packages/MPIHaloArrays/WDydQ/src/updatehalo.jl:351
[3] top-level scope
@ ~/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
in expression starting at /home/matt/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
6): MPI_ERR_RANK: invalid rank
Stacktrace:
[1] Isend(buf::MPI.Buffer{Matrix{Int64}}, dest::Int64, tag::Int64, comm::MPI.Comm)
@ MPI ~/.julia/packages/MPI/08SPr/src/pointtopoint.jl:229
[2] updatehalo!(A::MPIHaloArray{Int64, 2, Matrix{Int64}, 2})
@ MPIHaloArrays ~/.julia/packages/MPIHaloArrays/WDydQ/src/updatehalo.jl:352
[3] top-level scope
@ ~/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
in expression starting at /home/matt/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
MPI.Buffer{Matrix{Int64}}, dest::Int64, tag::Int64, comm::MPI.Comm)
@ MPI ~/.julia/packages/MPI/08SPr/src/pointtopoint.jl:229
[2] updatehalo!(A::MPIHaloArray{Int64, 2, Matrix{Int64}, 2})
@ MPIHaloArrays ~/.julia/packages/MPIHaloArrays/WDydQ/src/updatehalo.jl:351
[3] top-level scope
@ ~/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
in expression starting at /home/matt/Dropbox/Research/LLModel/src/multi_gpu/mpi_test.jl:48
--------------------------------------------------------------------------
Primary job terminated normally, but 1 process returned
a non-zero exit code. Per user-direction, the job has been aborted.
--------------------------------------------------------------------------
signal (15): Terminated
in expression starting at none:0
signal (15): Terminated
in expression starting at none:0
--------------------------------------------------------------------------
mpiexec detected that one or more processes exited with non-zero status, thus causing
the job to be terminated. The first process to do so was:
Process name: [[32345,1],1]
Exit code: 1
--------------------------------------------------------------------------
ERROR: failed process: Process(`/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/comm_libs/mpi/bin/mpiexec -n 8 julia mpi_test.jl`, ProcessExited(1)) [1]
Stacktrace:
[1] pipeline_error
@ ./process.jl:565 [inlined]
[2] run(::Cmd; wait::Bool)
@ Base ./process.jl:480
[3] run(::Cmd)
@ Base process.jl:477
[4] (::var"#1#2")(exe::Cmd)
@ Main none:4
[5] (::MPI.var"#28#29"{var"#1#2"})(cmd::Cmd)
@ MPI ~/.julia/packages/MPI/08SPr/src/environment.jl:25
[6] _mpiexec(fn::MPI.var"#28#29"{var"#1#2"})
@ MPI ~/.julia/packages/MPI/08SPr/deps/deps.jl:6
[7] mpiexec(fn::var"#1#2")
@ MPI ~/.julia/packages/MPI/08SPr/src/environment.jl:25
[8] top-level scope
@ none:4
Is there a function that reads a 3 dimensional array from process 1 and sends specific block to another process
@JuliaRegistrator register
This package looks great! It's simple and flexible enough for the kinds of finite-difference problems im interested in tackling.
Can I ask what your roadmap looks like? Have you tried performing large-scale analysis (100+ procs) to stress test this? Have you explored asynchronous halo updates when using eg GPUs? Do you have a list of tasks you'd like to work on, but don't have the bandwidth? Are there any "sharp bits" you are avoiding?
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.