Hi,
Congratulations with the development of Scenic+, it looks like a really useful tool and I can't wait to try it out.
Describe the bug
I am trying to run the tutorial notebooks on a HPC cluster, but this is causing problems with the ray parallelization for me. My notebooks run through an interactive SLURM job. Do you have any explanation available on how to get Ray to work in this context? Many thanks in advance.
To Reproduce
SLURM submission script:
#!/bin/bash
#SBATCH -p cclake
#SBATCH -A gottgens-sl2-cpu
#SBATCH -N 1
#SBATCH -n 12
get/set tunneling info
XDG_RUNTIME_DIR=""
ipnport=$(shuf -i8000-9999 -n1)
ipnip=$(hostname -i)
Start jupyter
jupyter-lab --no-browser --port=$ipnport --ip=$ipnip
Then in Jupyter-notebook ( this or any other command that has the n_pcu option) :
run_scenicplus(
scplus_obj = scplus_obj,
variable = ['GEX_celltype'],
species = 'hsapiens',
assembly = 'hg38',
tf_file = 'pbmc_tutorial/data/utoronto_human_tfs_v_1.01.txt',
save_path = os.path.join(work_dir, 'scenicplus'),
biomart_host = biomart_host,
upstream = [1000, 150000],
downstream = [1000, 150000],
calculate_TF_eGRN_correlation = True,
calculate_DEGs_DARs = True,
export_to_loom_file = True,
export_to_UCSC_file = True,
path_bedToBigBed = 'pbmc_tutorial',
n_cpu = 5,
_temp_dir = os.path.join(tmp_dir, 'ray_spill'))
It seems to run with n_cpu=None, but this takes a really long time.
Error output
2022-08-22 09:17:00,608 INFO services.py:1470 -- View the Ray dashboard at http://127.0.0.1:8267/
TimeoutError Traceback (most recent call last)
File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/ray/node.py:303, in Node.init(self, ray_params, head, shutdown_at_exit, spawn_reaper, connect_only)
302 try:
--> 303 ray._private.services.wait_for_node(
304 self.redis_address,
305 self.gcs_address,
306 self._plasma_store_socket_name,
307 self.redis_password,
308 )
309 except TimeoutError:
File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/ray/_private/services.py:397, in wait_for_node(redis_address, gcs_address, node_plasma_store_socket_name, redis_password, timeout)
396 time.sleep(0.1)
--> 397 raise TimeoutError("Timed out while waiting for node to startup.")
TimeoutError: Timed out while waiting for node to startup.
During handling of the above exception, another exception occurred:
Exception Traceback (most recent call last)
Input In [18], in <cell line: 2>()
20 except Exception as e:
21 #in case of failure, still save the object
22 dill.dump(scplus_obj, open(os.path.join(work_dir, 'scenicplus/scplus_obj.pkl'), 'wb'), protocol=-1)
---> 23 raise(e)
Input In [18], in <cell line: 2>()
1 from scenicplus.wrappers.run_scenicplus import run_scenicplus
2 try:
----> 3 run_scenicplus(
4 scplus_obj = scplus_obj,
5 variable = ['GEX_celltype'],
6 species = 'hsapiens',
7 assembly = 'hg38',
8 tf_file = 'pbmc_tutorial/data/utoronto_human_tfs_v_1.01.txt',
9 save_path = os.path.join(work_dir, 'scenicplus'),
10 biomart_host = biomart_host,
11 upstream = [1000, 150000],
12 downstream = [1000, 150000],
13 calculate_TF_eGRN_correlation = True,
14 calculate_DEGs_DARs = True,
15 export_to_loom_file = True,
16 export_to_UCSC_file = True,
17 path_bedToBigBed = 'pbmc_tutorial',
18 n_cpu = 5,
19 _temp_dir = os.path.join(tmp_dir, 'ray_spill'))
20 except Exception as e:
21 #in case of failure, still save the object
22 dill.dump(scplus_obj, open(os.path.join(work_dir, 'scenicplus/scplus_obj.pkl'), 'wb'), protocol=-1)
File /rds/project/rds-SDzz0CATGms/users/bt392/software/github/scenicplus/src/scenicplus/wrappers/run_scenicplus.py:141, in run_scenicplus(scplus_obj, variable, species, assembly, tf_file, save_path, biomart_host, upstream, downstream, region_ranking, gene_ranking, simplified_eGRN, calculate_TF_eGRN_correlation, calculate_DEGs_DARs, export_to_loom_file, export_to_UCSC_file, tree_structure, path_bedToBigBed, n_cpu, _temp_dir)
139 if 'region_to_gene' not in scplus_obj.uns.keys():
140 log.info('Inferring region to gene relationships')
--> 141 calculate_regions_to_genes_relationships(scplus_obj,
142 ray_n_cpu = n_cpu,
143 _temp_dir = _temp_dir,
144 importance_scoring_method = 'GBM',
145 importance_scoring_kwargs = GBM_KWARGS)
147 if 'TF2G_adj' not in scplus_obj.uns.keys():
148 log.info('Inferring TF to gene relationships')
File /rds/project/rds-SDzz0CATGms/users/bt392/software/github/scenicplus/src/scenicplus/enhancer_to_gene.py:661, in calculate_regions_to_genes_relationships(SCENICPLUS_obj, search_space_key, mask_expr_dropout, genes, importance_scoring_method, importance_scoring_kwargs, correlation_scoring_method, ray_n_cpu, add_distance, key_added, inplace, **kwargs)
658 log.info(
659 f'Calculating region to gene importances, using {importance_scoring_method} method')
660 start_time = time.time()
--> 661 region_to_gene_importances = _score_regions_to_genes(SCENICPLUS_obj,
662 search_space=search_space,
663 mask_expr_dropout=mask_expr_dropout,
664 genes=genes,
665 regressor_type=importance_scoring_method,
666 regressor_kwargs=importance_scoring_kwargs,
667 ray_n_cpu=ray_n_cpu,
668 **kwargs)
669 log.info('Took {} seconds'.format(time.time() - start_time))
671 # calculate region to gene correlation
File /rds/project/rds-SDzz0CATGms/users/bt392/software/github/scenicplus/src/scenicplus/enhancer_to_gene.py:534, in _score_regions_to_genes(SCENICPLUS_obj, search_space, mask_expr_dropout, genes, regressor_type, ray_n_cpu, regressor_kwargs, **kwargs)
532 ACC_df = SCENICPLUS_obj.to_df(layer='ACC')
533 if ray_n_cpu != None:
--> 534 ray.init(num_cpus=ray_n_cpu, **kwargs)
535 try:
536 jobs = []
File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/ray/_private/client_mode_hook.py:105, in client_mode_hook..wrapper(*args, **kwargs)
103 if func.name != "init" or is_client_mode_enabled_by_default:
104 return getattr(ray, func.name)(*args, **kwargs)
--> 105 return func(*args, **kwargs)
File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/ray/worker.py:1039, in init(address, num_cpus, num_gpus, resources, object_store_memory, local_mode, ignore_reinit_error, include_dashboard, dashboard_host, dashboard_port, job_config, configure_logging, logging_level, logging_format, log_to_driver, namespace, runtime_env, storage, _enable_object_reconstruction, _redis_max_memory, _plasma_directory, _node_ip_address, _driver_object_store_memory, _memory, _redis_password, _temp_dir, _metrics_export_port, _system_config, _tracing_startup_hook, _node_name, **kwargs)
997 ray_params = ray._private.parameter.RayParams(
998 node_ip_address=node_ip_address,
999 raylet_ip_address=raylet_ip_address,
(...)
1033 node_name=_node_name,
1034 )
1035 # Start the Ray processes. We set shutdown_at_exit=False because we
1036 # shutdown the node in the ray.shutdown call that happens in the atexit
1037 # handler. We still spawn a reaper process in case the atexit handler
1038 # isn't called.
-> 1039 _global_node = ray.node.Node(
1040 head=True, shutdown_at_exit=False, spawn_reaper=True, ray_params=ray_params
1041 )
1042 else:
1043 # In this case, we are connecting to an existing cluster.
1044 if num_cpus is not None or num_gpus is not None:
File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/ray/node.py:310, in Node.init(self, ray_params, head, shutdown_at_exit, spawn_reaper, connect_only)
303 ray._private.services.wait_for_node(
304 self.redis_address,
305 self.gcs_address,
306 self._plasma_store_socket_name,
307 self.redis_password,
308 )
309 except TimeoutError:
--> 310 raise Exception(
311 "The current node has not been updated within 30 "
312 "seconds, this could happen because of some of "
313 "the Ray processes failed to startup."
314 )
315 node_info = ray._private.services.get_node_to_connect_for_driver(
316 self.redis_address,
317 self.gcs_address,
318 self._raylet_ip_address,
319 redis_password=self.redis_password,
320 )
321 self._ray_params.node_manager_port = node_info.node_manager_port
Exception: The current node has not been updated within 30 seconds, this could happen because of some of the Ray processes failed to startup.
Version (please complete the following information):
- Python: 3.8.13
- SCENIC+: 0.1.dev435+g5987737.d20220821
- Ray 3.8.13
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent