Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enable meta parallel Circuitscape - call Circuitscape on several INI files at once with 1 process each #248

Closed
wants to merge 5 commits into from
Closed

Enable meta parallel Circuitscape - call Circuitscape on several INI files at once with 1 process each #248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fcb6607
Enable meta parallelism
ranjanan Jan 23, 2019
f64a281
Get it to work
ranjanan Jan 24, 2019
410600e
Merge branch 'master' into RA/metapar
ranjanan Jun 16, 2020
f58b7d1
Remove unnecessary metaparallelize flag
ranjanan Jun 16, 2020
d0c117e
Reflect change in the meta_parallel function
ranjanan Jun 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/config.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,11 @@ function init_config()
a["log_file"] = "None"
a["log_level"] = "INFO"
a["cholmod_batch_size"] = "1000"
a["use_64bit_indexing"] = "false"
a["use_64bit_indexing"] = "true"
a["write_as_tif"] = "false"
a["meta_parallelize"] = "false"

a
end

Expand Down
7 changes: 6 additions & 1 deletion src/core.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ function amg_solver_path(data::GraphData{T,V}, flags, cfg, log)::Matrix{T} where
write_cur_maps = outputflags.write_cur_maps
write_cum_cur_map_only = outputflags.write_cum_cur_map_only
write_max_cur_maps = outputflags.write_max_cur_maps
meta_parallelize = flags.meta_parallelize

# Get number of focal points
numpoints = size(points, 1)
Expand Down Expand Up @@ -110,6 +111,8 @@ function amg_solver_path(data::GraphData{T,V}, flags, cfg, log)::Matrix{T} where
csinfo("Total number of pair solves has been reduced to $num ")
end
shortcut = Shortcut(get_shortcut_resistances, voltmatrix, shortcut_res)

mapf = meta_parallelize ? map : pmap

for (cid, comp) in enumerate(cc)

Expand Down Expand Up @@ -279,6 +282,7 @@ function _cholmod_solver_path(data::GraphData{T,V}, flags,
write_cur_maps = outputflags.write_cur_maps
write_cum_cur_map_only = outputflags.write_cum_cur_map_only
write_max_cur_maps = outputflags.write_max_cur_maps
meta_parallelize = flags.meta_parallelize

# Cumulative current map
cum = data.cum
Expand Down Expand Up @@ -316,6 +320,8 @@ function _cholmod_solver_path(data::GraphData{T,V}, flags,
end
shortcut = Shortcut(get_shortcut_resistances, voltmatrix, shortcut_res)

mapf = meta_parallelize ? map : pmap

for (cid, comp) in enumerate(cc)

# Subset of points relevant to CC
Expand Down Expand Up @@ -421,7 +427,6 @@ function _cholmod_solver_path(data::GraphData{T,V}, flags,
lhs[j,i] = lhs[j,i] - v
end
end

is_parallel = cfg["parallelize"] in TRUELIST
if is_parallel
X = pmap(x -> f(x, rng, lhs), 1:length(rng))
Expand Down
1 change: 1 addition & 0 deletions src/network/pairwise.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ function get_network_flags(cfg)
is_advanced = cfg["scenario"] in ADVANCED
is_alltoone = false
is_onetoall = false
meta_parallelize = cfg["meta_parallelize"] in TRUELIST
grnd_file_is_res = cfg["ground_file_is_resistances"] in TRUELIST
policy = Symbol(cfg["remove_src_or_gnd"])
solver = cfg["solver"]
Expand Down
4 changes: 3 additions & 1 deletion src/raster/pairwise.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ struct RasterFlags
is_advanced::Bool
is_onetoall::Bool
is_alltoone::Bool
meta_parallelize::Bool
grnd_file_is_res::Bool
policy::Symbol
four_neighbors::Bool
Expand Down Expand Up @@ -44,12 +45,13 @@ function get_raster_flags(cfg)
ground_file_is_resistances =
cfg["ground_file_is_resistances"] in TRUELIST
policy = Symbol(cfg["remove_src_or_gnd"])
meta_parallelize = cfg["meta_parallelize"] in TRUELIST

# Output Flags
o = get_output_flags(cfg)

RasterFlags(is_raster, is_pairwise, is_advanced,
is_onetoall, is_alltoone,
is_onetoall, is_alltoone, meta_parallelize,
ground_file_is_resistances, policy,
four_neighbors, avg_res, solver, o)
end
Expand Down
2 changes: 1 addition & 1 deletion src/run.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
export compute
export compute, compute_metaparallel

"""
`compute(path::String)`
Expand Down
19 changes: 19 additions & 0 deletions src/utils.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,25 @@ function compute_parallel(str, n_processes = 2)
compute(cfg)
end

function compute_metaparallel(jobs::Vector{String}, n_procs = 1)

l = length(jobs)
csinfo("Starting up $(n_procs) processes")
myaddprocs(n_procs)
csinfo("Launching $l Circuitscape jobs among $(n_procs) processes")

cfg_list = Vector{Dict{String,String}}(undef,l)
for (i,str) in enumerate(jobs)
cfg = parse_config(str)
cfg["parallelize"] = "false"
cfg_list[i] = cfg
end
@everywhere Core.eval(Main, :(using Circuitscape))
ret = pmap(compute, cfg_list)
rmprocs(workers())
ret
end

function get_output_flags(cfg)

# Output flags
Expand Down