denisecailab / minian Goto Github PK
View Code? Open in Web Editor NEWminiscope analysis pipeline with interactive visualizations
License: GNU General Public License v3.0
miniscope analysis pipeline with interactive visualizations
License: GNU General Public License v3.0
Hi,
I would like to use multiple CPUs for faster analysis, and would like to know how to modify the jupyter notebook script?
Thanks!
Hi,
When running the pipeline_noted.ipynb with the demo movies I get the following error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-11-24aa6b1de449> in <module>
1 hv.output(size=output_size)
2 if interactive:
----> 3 display(visualize_preprocess(varr_ref.isel(frame=0), denoise, method=['median'], ksize=[5, 7, 9]))
~/Downloads/minian-master/minian/visualization.py in visualize_preprocess(fm, fn, include_org, **kwargs)
1365 for params in itt.product(*pval):
1366 fm_res = fn(fm, **dict(zip(pkey, params)))
-> 1367 cur_im, cur_cnt = _vis(fm_res)
1368 cur_im = cur_im.relabel('After')
1369 cur_cnt = cur_cnt.relabel('After')
~/Downloads/minian-master/minian/visualization.py in _vis(f)
1355 im = (hv.Image(f, kdims=['width', 'height'])
1356 .opts(**opts_im))
-> 1357 cnt = (hv.operation.contours(im)
1358 .opts(**opts_cnt))
1359 return im, cnt
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/param/parameterized.py in __new__(class_, *args, **params)
2810 inst = class_.instance()
2811 inst.param._set_name(class_.__name__)
-> 2812 return inst.__call__(*args,**params)
2813
2814 def __call__(self,*args,**kw):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/operation.py in __call__(self, element, **kwargs)
162 elif 'streams' not in kwargs:
163 kwargs['streams'] = self.p.streams
--> 164 return element.apply(self, **kwargs)
165
166
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/accessors.py in __call__(self, function, streams, link_inputs, dynamic, **kwargs)
108 if hasattr(function, 'dynamic'):
109 inner_kwargs['dynamic'] = False
--> 110 return function(self._obj, **inner_kwargs)
111 elif self._obj._deep_indexable:
112 mapped = []
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/operation.py in __call__(self, element, **kwargs)
159 for k, el in element.items()])
160 elif isinstance(element, ViewableElement):
--> 161 return self._apply(element)
162 elif 'streams' not in kwargs:
163 kwargs['streams'] = self.p.streams
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/operation.py in _apply(self, element, key)
119 for hook in self._preprocess_hooks:
120 kwargs.update(hook(self, element))
--> 121 ret = self._process(element, key)
122 for hook in self._postprocess_hooks:
123 ret = hook(self, ret, **kwargs)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/operation/element.py in _process(self, element, key)
433 xs = element.dimension_values(0, True, flat=False)
434 ys = element.dimension_values(1, True, flat=False)
--> 435 zs = element.dimension_values(2, flat=False)
436
437 # Ensure that coordinate arrays specify bin centers
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/data/__init__.py in dimension_values(self, dimension, expanded, flat)
768 """
769 dim = self.get_dimension(dimension, strict=True)
--> 770 return self.interface.values(self, dim, expanded, flat)
771
772
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/data/xarray.py in values(cls, dataset, dim, expanded, flat, compute)
325 da = dask_array_module()
326 if compute and da and isinstance(data, da.Array):
--> 327 data = data.compute()
328 data = cls.canonicalize(dataset, data, data_coords=data_coords,
329 virtual_coords=virtual_coords)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/base.py in compute(self, **kwargs)
154 dask.base.compute
155 """
--> 156 (result,) = compute(self, traverse=False, **kwargs)
157 return result
158
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/base.py in compute(*args, **kwargs)
396 keys = [x.__dask_keys__() for x in collections]
397 postcomputes = [x.__dask_postcompute__() for x in collections]
--> 398 results = schedule(dsk, keys, **kwargs)
399 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
400
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, pool, **kwargs)
74 results = get_async(pool.apply_async, len(pool._pool), dsk, result,
75 cache=cache, get_id=_thread_get_id,
---> 76 pack_exception=pack_exception, **kwargs)
77
78 # Cleanup pools associated to dead threads
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
460 _execute_task(task, data) # Re-execute locally
461 else:
--> 462 raise_exception(exc, tb)
463 res, worker_id = loads(res_info)
464 state['cache'][key] = res
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/compatibility.py in reraise(exc, tb)
110 if exc.__traceback__ is not tb:
111 raise exc.with_traceback(tb)
--> 112 raise exc
113
114 import pickle as cPickle
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
228 try:
229 task, data = loads(task_info)
--> 230 result = _execute_task(task, data)
231 id = get_id()
232 result = dumps((result, id))
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
117 func, args = arg[0], arg[1:]
118 args2 = [_execute_task(a, cache) for a in args]
--> 119 return func(*args2)
120 elif not ishashable(arg):
121 return arg
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/optimization.py in __call__(self, *args)
940 % (len(self.inkeys), len(args)))
941 return core.get(self.dsk, self.outkey,
--> 942 dict(zip(self.inkeys, args)))
943
944 def __reduce__(self):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/core.py in get(dsk, out, cache)
147 for key in toposort(dsk):
148 task = dsk[key]
--> 149 result = _execute_task(task, cache)
150 cache[key] = result
151 result = _execute_task(out, cache)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
117 func, args = arg[0], arg[1:]
118 args2 = [_execute_task(a, cache) for a in args]
--> 119 return func(*args2)
120 elif not ishashable(arg):
121 return arg
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/dask/compatibility.py in apply(func, args, kwargs)
91 def apply(func, args, kwargs=None):
92 if kwargs:
---> 93 return func(*args, **kwargs)
94 else:
95 return func(*args)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/numpy/lib/function_base.py in __call__(self, *args, **kwargs)
2089 vargs.extend([kwargs[_n] for _n in names])
2090
-> 2091 return self._vectorize_call(func=func, args=vargs)
2092
2093 def _get_ufunc_and_otypes(self, func, args):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/numpy/lib/function_base.py in _vectorize_call(self, func, args)
2155 """Vectorized call to `func` over positional `args`."""
2156 if self.signature is not None:
-> 2157 res = self._vectorize_call_with_signature(func, args)
2158 elif not args:
2159 res = func()
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/numpy/lib/function_base.py in _vectorize_call_with_signature(self, func, args)
2181 raise TypeError('wrong number of positional arguments: '
2182 'expected %r, got %r'
-> 2183 % (len(input_core_dims), len(args)))
2184 args = tuple(asanyarray(arg) for arg in args)
2185
TypeError: wrong number of positional arguments: expected 1, got 2
Could it be a bug due to a new version of dask/numpy like here 'fix a vectorize bug from new dask/numpy version'? Using the docker isn´t working either.
Many thanks in advance!
I am having issues when I get to the CNMF step of the pipeline both with the demo and with my own data. When I am running the kernels under "test parameters for spatial update" or "test parameters for temporal update" the kernel gets stuck and never finishes. It will stop at some percentage and the timer will continue to count, but no progress will be made.
It does not occur at the same kernel every time, and when it is able to complete the kernel, it finishes relatively quickly. I have tried letting it run for 48 hours after getting stuck and it never finishes. This has occurred on multiple powerful machines (both windows and mac). I also tried running it on linux through a virtual box within a windows machine, and the same thing happened, although it got further (got stuck on the 2nd temporal update).
I never get an error message, the kernel just becomes unresponsive. Any idea what might be causing this?
I am running on the dev branch as recommended.
Thanks!!
Hi--
I love the package overall. I tried sharing the pipeline with lab mates by installing on a few new computers (PCs, Windows 10).
When attempting to install the pipeline for the first time on these new PC's, and verifying the installation using "pipeline_noted" notebook, I receive an error when trying to import the minian modules.
I've included an image of the code and error. Can you please advice how to solve this issue?
I am working with long recording sessions (more than 25 files with 1000 frames) and I have found that minian cannot process them all in one chunk as I face memory errors even when allocating 1.5TB of RAM to the process.
So I have now broken down my session to 3 chunks and have successfully run the pipeline on all and I need to cross-register them now. But unfortunately, I cannot get this notebook to work (I have tried both master and dev branches). The current script on the master branch seems to be functioning with less errors however I still get this error:
My own guess is that the problem comes from when I am opening the datasets:
As you can see, I am opening 3 separate folders of data but apparently, only the last folder has been stored in minian_ds.
I don't set any of the data dimensions manually when I run minian on my videos; Could the problem be because of the naming of the sessions or other data dimensions?
Am I missing something here? Has anyone been able to successfully do a cross-registration on different chunks of a long session?
Hi @phildong
it will be very helpful if you drop few lines of text about where is your demo movie came from.
I've noticed two things:
Regards,
Denis.
Better done after last item in #14 . Alternatively, the methods can be bench marked by "exclusivity" and "transitivity" (Ref CellReg). Ideas:
Having a separate pre-batch notebook file for users who wants to select ROI for every video will be helpful! Especially when imaging is not clean which have dirts on either lens or miniscope. Generate an output file for ROI info in each folder, if users don't need to set different ROIs for their videos (don't have that ROI info file in the video folder), can automatically use the default for motion correction.
should be easy to implement with holoviews' box draw stream
http://holoviews.org/gallery/demos/bokeh/box_draw_roi_editor.html
implement some metric of motion correction. Ideas:
Hi,
Recently, from about 1month ago, I start to have a problem making videos(concatenated) with some error message. It seemed like it is ffmpeg related problem, I tried to fix it by downloading ffmpeg again but it did not work, so I want to report it to minian developers. I am using vlc player in Linux ubuntu.
Here I post my error screenshot, with the final product minian.avi file with has white background only (unlike before which used to have 4 divided screens).
And another question: is there any way to change the code in my local computer so that I can generate my final videos after manual curation (delete or merge)? Now it is 'generate video'- 'curation'- then 'save new unit IDs', but these deleted or merged units are not fixed in the video or output display which has been already generated. Could you let me know which part I should change to get them after curation? (Or to plot cells that I am interested in only.)
Thanks!
Hey all!
The pipeline has been working great in terms of data processing, however I have had a significant issue with the visualization necessary to properly define the parameters of the model.
The problem looks like a hex to rgb conversion issue that I've been getting in the denoise and background removal displays. It seems like the function expects to get one value instead of a list, but I am unsure of how to go about fixing this. I have the most recent versions of holoviews, datashader, and panel that are called in the error.
TypeError Traceback (most recent call last)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/IPython/core/formatters.py in call(self, obj, include, exclude)
968
969 if method is not None:
--> 970 return method(include=include, exclude=exclude)
971 return None
972 else:
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/dimension.py in repr_mimebundle(self, include, exclude)
1302 combined and returned.
1303 """
-> 1304 return Store.render(self)
1305
1306
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/options.py in render(cls, obj)
1393 data, metadata = {}, {}
1394 for hook in hooks:
-> 1395 ret = hook(obj)
1396 if ret is None:
1397 continue
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/ipython/display_hooks.py in pprint_display(obj)
280 if not ip.display_formatter.formatters['text/plain'].pprint:
281 return None
--> 282 return display(obj, raw_output=True)
283
284
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/ipython/display_hooks.py in display(obj, raw_output, **kwargs)
253 elif isinstance(obj, (Layout, NdLayout, AdjointLayout)):
254 with option_state(obj):
--> 255 output = layout_display(obj)
256 elif isinstance(obj, (HoloMap, DynamicMap)):
257 with option_state(obj):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/ipython/display_hooks.py in wrapped(element)
144 try:
145 max_frames = OutputSettings.options['max_frames']
--> 146 mimebundle = fn(element, max_frames=max_frames)
147 if mimebundle is None:
148 return {}, {}
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/ipython/display_hooks.py in layout_display(layout, max_frames)
218 return None
219
--> 220 return render(layout)
221
222
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/ipython/display_hooks.py in render(obj, **kwargs)
66 renderer = renderer.instance(fig='png')
67
---> 68 return renderer.components(obj, **kwargs)
69
70
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/plotting/renderer.py in components(self, obj, fmt, comm, **kwargs)
374 doc = Document()
375 with config.set(embed=embed):
--> 376 model = plot.layout._render_model(doc, comm)
377 if embed:
378 return render_model(model, comm)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/panel/viewable.py in _render_model(self, doc, comm)
415 if comm is None:
416 comm = state._comm_manager.get_server_comm()
--> 417 model = self.get_root(doc, comm)
418
419 if config.embed:
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/panel/viewable.py in get_root(self, doc, comm)
640 """
641 doc = doc or _curdoc()
--> 642 root = self._get_model(doc, comm=comm)
643 self._preprocess(root)
644 ref = root.ref['id']
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/panel/layout.py in _get_model(self, doc, root, parent, comm)
118 if root is None:
119 root = model
--> 120 objects = self._get_objects(model, [], doc, root, comm)
121 props = dict(self._init_properties(), objects=objects)
122 model.update(**self._process_param_change(props))
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/panel/layout.py in _get_objects(self, model, old_objects, doc, root, comm)
108 else:
109 try:
--> 110 child = pane._get_model(doc, root, model, comm)
111 except RerenderError:
112 return self._get_objects(model, current_objects[:i], doc, root, comm)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/panel/pane/holoviews.py in _get_model(self, doc, root, parent, comm)
225 plot = self.object
226 else:
--> 227 plot = self._render(doc, comm, root)
228
229 plot.pane = self
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/panel/pane/holoviews.py in _render(self, doc, comm, root)
284 kwargs = {}
285
--> 286 return renderer.get_plot(self.object, **kwargs)
287
288 def _cleanup(self, root):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/plotting/bokeh/renderer.py in get_plot(self_or_cls, obj, doc, renderer, **kwargs)
71 combining the bokeh model with another plot.
72 """
---> 73 plot = super(BokehRenderer, self_or_cls).get_plot(obj, doc, renderer, **kwargs)
74 if plot.document is None:
75 plot.document = Document() if self_or_cls.notebook_context else curdoc()
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/plotting/renderer.py in get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs)
209
210 # Initialize DynamicMaps with first data item
--> 211 initialize_dynamic(obj)
212
213 if not renderer:
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/plotting/util.py in initialize_dynamic(obj)
249 continue
250 if not len(dmap):
--> 251 dmap[dmap._initial_key()]
252
253
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in getitem(self, key)
1278 # Not a cross product and nothing cached so compute element.
1279 if cache is not None: return cache
-> 1280 val = self._execute_callback(*tuple_key)
1281 if data_slice:
1282 val = self._dataslice(val, data_slice)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args)
1052
1053 with dynamicmap_memoization(self.callback, self.streams):
-> 1054 retval = self.callback(*args, **kwargs)
1055 return self._style(retval)
1056
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in call(self, *args, **kwargs)
691
692 try:
--> 693 ret = self.callable(*args, **kwargs)
694 except KeyError:
695 # KeyError is caught separately because it is used to signal
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/util/init.py in dynamic_operation(*key, **kwargs)
983
984 def dynamic_operation(*key, **kwargs):
--> 985 key, obj = resolve(key, kwargs)
986 return apply(obj, *key, **kwargs)
987
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/util/init.py in resolve(key, kwargs)
976 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key:
977 key = tuple(kwargs[k] for k in map_obj._posarg_keys)
--> 978 return key, map_obj[key]
979
980 def apply(element, *key, **kwargs):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in getitem(self, key)
1278 # Not a cross product and nothing cached so compute element.
1279 if cache is not None: return cache
-> 1280 val = self._execute_callback(*tuple_key)
1281 if data_slice:
1282 val = self._dataslice(val, data_slice)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args)
1052
1053 with dynamicmap_memoization(self.callback, self.streams):
-> 1054 retval = self.callback(*args, **kwargs)
1055 return self._style(retval)
1056
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in call(self, *args, **kwargs)
691
692 try:
--> 693 ret = self.callable(*args, **kwargs)
694 except KeyError:
695 # KeyError is caught separately because it is used to signal
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/util/init.py in dynamic_operation(*key, **kwargs)
983
984 def dynamic_operation(*key, **kwargs):
--> 985 key, obj = resolve(key, kwargs)
986 return apply(obj, *key, **kwargs)
987
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/util/init.py in resolve(key, kwargs)
976 elif isinstance(map_obj, DynamicMap) and map_obj._posarg_keys and not key:
977 key = tuple(kwargs[k] for k in map_obj._posarg_keys)
--> 978 return key, map_obj[key]
979
980 def apply(element, *key, **kwargs):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in getitem(self, key)
1278 # Not a cross product and nothing cached so compute element.
1279 if cache is not None: return cache
-> 1280 val = self._execute_callback(*tuple_key)
1281 if data_slice:
1282 val = self._dataslice(val, data_slice)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in _execute_callback(self, *args)
1052
1053 with dynamicmap_memoization(self.callback, self.streams):
-> 1054 retval = self.callback(*args, **kwargs)
1055 return self._style(retval)
1056
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/spaces.py in call(self, *args, **kwargs)
691
692 try:
--> 693 ret = self.callable(*args, **kwargs)
694 except KeyError:
695 # KeyError is caught separately because it is used to signal
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/util/init.py in dynamic_operation(*key, **kwargs)
984 def dynamic_operation(*key, **kwargs):
985 key, obj = resolve(key, kwargs)
--> 986 return apply(obj, *key, **kwargs)
987
988 operation = self.p.operation
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/util/init.py in apply(element, *key, **kwargs)
980 def apply(element, *key, **kwargs):
981 kwargs = dict(util.resolve_dependent_kwargs(self.p.kwargs), **kwargs)
--> 982 return self._process(element, key, kwargs)
983
984 def dynamic_operation(*key, **kwargs):
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/util/init.py in _process(self, element, key, kwargs)
962 elif isinstance(self.p.operation, Operation):
963 kwargs = {k: v for k, v in kwargs.items() if k in self.p.operation.param}
--> 964 return self.p.operation.process_element(element, key, **kwargs)
965 else:
966 return self.p.operation(element, **kwargs)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/operation.py in process_element(self, element, key, **params)
170 self.p = param.ParamOverrides(self, params,
171 allow_extra_keywords=self._allow_extra_keywords)
--> 172 return self._apply(element, key)
173
174
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/core/operation.py in _apply(self, element, key)
130 element_pipeline = getattr(element, '_pipeline', None)
131
--> 132 ret = self._process(element, key)
133 for hook in self._postprocess_hooks:
134 ret = hook(self, ret, **kwargs)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/operation/datashader.py in _process(self, element, key)
1435 def _process(self, element, key=None):
1436 agg = rasterize._process(self, element, key)
-> 1437 shaded = shade._process(self, agg, key)
1438 return shaded
1439
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/holoviews/operation/datashader.py in _process(self, element, key)
1267 return RGB(img, **params)
1268 else:
-> 1269 img = tf.shade(array, **shade_opts)
1270 return RGB(self.uint32_to_uint8_xr(img), **params)
1271
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/datashader/transfer_functions/init.py in shade(agg, cmap, color_key, how, alpha, min_alpha, span, name, color_baseline)
509
510 if agg.ndim == 2:
--> 511 return _interpolate(agg, cmap, how, alpha, span, min_alpha, name)
512 elif agg.ndim == 3:
513 return _colorize(agg, color_key, how, alpha, span, min_alpha, name, color_baseline)
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/datashader/transfer_functions/init.py in _interpolate(agg, cmap, how, alpha, span, min_alpha, name)
265 rgba = np.dstack([r, g, b, a])
266 elif isinstance(cmap, str) or isinstance(cmap, tuple):
--> 267 color = rgb(cmap)
268 aspan = np.arange(min_alpha, alpha+1)
269 span = np.linspace(span[0], span[1], len(aspan))
~/opt/anaconda3/envs/minian/lib/python3.7/site-packages/datashader/colors.py in rgb(x)
129 raise ValueError("Invalid RGB tuple")
130 else:
--> 131 raise TypeError("Don't know how to convert {0} to RGB".format(x))
132 return x
133
TypeError: Don't know how to convert ('#440154', '#440255', '#440357', '#450558', '#45065A', '#45085B', '#46095C', '#460B5E', '#460C5F', '#460E61', '#470F62', '#471163', '#471265', '#471466', '#471567', '#471669', '#47186A', '#48196B', '#481A6C', '#481C6E', '#481D6F', '#481E70', '#482071', '#482172', '#482273', '#482374', '#472575', '#472676', '#472777', '#472878', '#472A79', '#472B7A', '#472C7B', '#462D7C', '#462F7C', '#46307D', '#46317E', '#45327F', '#45347F', '#453580', '#453681', '#443781', '#443982', '#433A83', '#433B83', '#433C84', '#423D84', '#423E85', '#424085', '#414186', '#414286', '#404387', '#404487', '#3F4587', '#3F4788', '#3E4888', '#3E4989', '#3D4A89', '#3D4B89', '#3D4C89', '#3C4D8A', '#3C4E8A', '#3B508A', '#3B518A', '#3A528B', '#3A538B', '#39548B', '#39558B', '#38568B', '#38578C', '#37588C', '#37598C', '#365A8C', '#365B8C', '#355C8C', '#355D8C', '#345E8D', '#345F8D', '#33608D', '#33618D', '#32628D', '#32638D', '#31648D', '#31658D', '#31668D', '#30678D', '#30688D', '#2F698D', '#2F6A8D', '#2E6B8E', '#2E6C8E', '#2E6D8E', '#2D6E8E', '#2D6F8E', '#2C708E', '#2C718E', '#2C728E', '#2B738E', '#2B748E', '#2A758E', '#2A768E', '#2A778E', '#29788E', '#29798E', '#287A8E', '#287A8E', '#287B8E', '#277C8E', '#277D8E', '#277E8E', '#267F8E', '#26808E', '#26818E', '#25828E', '#25838D', '#24848D', '#24858D', '#24868D', '#23878D', '#23888D', '#23898D', '#22898D', '#228A8D', '#228B8D', '#218C8D', '#218D8C', '#218E8C', '#208F8C', '#20908C', '#20918C', '#1F928C', '#1F938B', '#1F948B', '#1F958B', '#1F968B', '#1E978A', '#1E988A', '#1E998A', '#1E998A', '#1E9A89', '#1E9B89', '#1E9C89', '#1E9D88', '#1E9E88', '#1E9F88', '#1EA087', '#1FA187', '#1FA286', '#1FA386', '#20A485', '#20A585', '#21A685', '#21A784', '#22A784', '#23A883', '#23A982', '#24AA82', '#25AB81', '#26AC81', '#27AD80', '#28AE7F', '#29AF7F', '#2AB07E', '#2BB17D', '#2CB17D', '#2EB27C', '#2FB37B', '#30B47A', '#32B57A', '#33B679', '#35B778', '#36B877', '#38B976', '#39B976', '#3BBA75', '#3DBB74', '#3EBC73', '#40BD72', '#42BE71', '#44BE70', '#45BF6F', '#47C06E', '#49C16D', '#4BC26C', '#4DC26B', '#4FC369', '#51C468', '#53C567', '#55C666', '#57C665', '#59C764', '#5BC862', '#5EC961', '#60C960', '#62CA5F', '#64CB5D', '#67CC5C', '#69CC5B', '#6BCD59', '#6DCE58', '#70CE56', '#72CF55', '#74D054', '#77D052', '#79D151', '#7CD24F', '#7ED24E', '#81D34C', '#83D34B', '#86D449', '#88D547', '#8BD546', '#8DD644', '#90D643', '#92D741', '#95D73F', '#97D83E', '#9AD83C', '#9DD93A', '#9FD938', '#A2DA37', '#A5DA35', '#A7DB33', '#AADB32', '#ADDC30', '#AFDC2E', '#B2DD2C', '#B5DD2B', '#B7DD29', '#BADE27', '#BDDE26', '#BFDF24', '#C2DF22', '#C5DF21', '#C7E01F', '#CAE01E', '#CDE01D', '#CFE11C', '#D2E11B', '#D4E11A', '#D7E219', '#DAE218', '#DCE218', '#DFE318', '#E1E318', '#E4E318', '#E7E419', '#E9E419', '#ECE41A', '#EEE51B', '#F1E51C', '#F3E51E', '#F6E61F', '#F8E621', '#FAE622', '#FDE724') to RGB
:Layout
.DynamicMap.Before.I :DynamicMap []
:Image [width,height] (fluorescence_denoised)
.DynamicMap.Before.II :DynamicMap []
.DynamicMap.After.I :DynamicMap [method,wnd]
.DynamicMap.After.II :DynamicMap [method,wnd]
The cross-registration notebook gives an error in the "Load Modules" cell because cross_registration.py
tries to import estimate_shift_fft
from motion_correction.py
but that function does not exist:
Changing estimate_shift_fft
to estimate_shifts
fixes this issue but then the next cell produces an error that seems related to the id_dims
parameter:
id_dims
is a list of two strings (id_dims = ['animal','session']
), and changing it to a list of one string (id_dims = ['session']
) creates a third error:
Is this an issue with the open_minian_mf()
function?
When running first temporal update, the terminal keeps printing:
frame= 651 fps= 41 q=29.0 size= 11520kB time=00:00:19.73 bitrate=4782.4kbits/
frame= 674 fps= 41 q=29.0 size= 12032kB time=00:00:20.50 bitrate=4808.1kbits/
frame= 698 fps= 41 q=29.0 size= 12544kB time=00:00:21.30 bitrate=4824.4kbits/
frame= 721 fps= 41 q=29.0 size= 13056kB time=00:00:22.06 bitrate=4846.9kbits/
frame= 744 fps= 41 q=29.0 size= 13312kB time=00:00:22.83 bitrate=4776.0kbits/
frame= 767 fps= 42 q=29.0 size= 13824kB time=00:00:23.60 bitrate=4798.6kbits/
frame= 790 fps= 42 q=29.0 size= 14336kB time=00:00:24.36 bitrate=4819.7kbits/
frame= 815 fps= 42 q=29.0 size= 14848kB time=00:00:25.20 bitrate=4826.8kbits/
frame= 836 fps= 42 q=29.0 size= 15360kB time=00:00:25.90 bitrate=4858.3kbits/
frame= 859 fps= 42 q=29.0 size= 15616kB time=00:00:26.66 bitrate=4797.2kbits/
frame= 881 fps= 42 q=29.0 size= 16128kB time=00:00:27.40 bitrate=4821.9kbits/
frame= 903 fps= 42 q=29.0 size= 16640kB time=00:00:28.13 bitrate=4845.3kbits/
frame= 926 fps= 42 q=29.0 size= 17152kB time=00:00:28.90 bitrate=4861.9kbits/
frame= 948 fps= 42 q=29.0 size= 17408kB time=00:00:29.63 bitrate=4812.4kbits/
frame= 971 fps= 42 q=29.0 size= 17920kB time=00:00:30.40 bitrate=4829.0kbits/
frame= 993 fps= 42 q=29.0 size= 18432kB time=00:00:31.13 bitrate=4849.9kbits/
frame= 1015 fps= 42 q=29.0 size= 18688kB time=00:00:31.86 bitrate=4804.1kbits/
frame= 1037 fps= 42 q=29.0 size= 19200kB time=00:00:32.60 bitrate=4824.7kbits/
frame= 1063 fps= 42 q=29.0 size= 19712kB time=00:00:33.46 bitrate=4825.1kbits/
frame= 1087 fps= 42 q=29.0 size= 20224kB time=00:00:34.26 bitrate=4834.9kbits/
frame= 1111 fps= 42 q=29.0 size= 20736kB time=00:00:35.06 bitrate=4844.2kbits/
frame= 1134 fps= 42 q=29.0 size= 21248kB time=00:00:35.83 bitrate=4857.6kbits/
frame= 1156 fps= 42 q=29.0 size= 21760kB time=00:00:36.56 bitrate=4874.9kbits/
frame= 1179 fps= 43 q=29.0 size= 22016kB time=00:00:37.33 bitrate=4830.9kbits/
frame= 1202 fps= 43 q=29.0 size= 22528kB time=00:00:38.10 bitrate=4843.8kbits/
frame= 1224 fps= 43 q=29.0 size= 23040kB time=00:00:38.83 bitrate=4860.4kbits/
frame= 1247 fps= 43 q=29.0 size= 23552kB time=00:00:39.60 bitrate=4872.2kbits/
frame= 1269 fps= 43 q=29.0 size= 23808kB time=00:00:40.33 bitrate=4835.6kbits/
frame= 1293 fps= 43 q=29.0 size= 24320kB time=00:00:41.13 bitrate=4843.5kbits/
frame= 1317 fps= 43 q=29.0 size= 24832kB time=00:00:41.93 bitrate=4851.1kbits/
frame= 1341 fps= 43 q=29.0 size= 25344kB time=00:00:42.73 bitrate=4858.5kbits/
frame= 1363 fps= 43 q=29.0 size= 25856kB time=00:00:43.46 bitrate=4873.0kbits/
frame= 1386 fps= 43 q=29.0 size= 26112kB time=00:00:44.23 bitrate=4835.9kbits/
frame= 1409 fps= 43 q=29.0 size= 26624kB time=00:00:45.00 bitrate=4846.8kbits/
frame= 1431 fps= 43 q=29.0 size= 27136kB time=00:00:45.73 bitrate=4860.7kbits/
frame= 1454 fps= 43 q=29.0 size= 27648kB time=00:00:46.50 bitrate=4870.8kbits/
frame= 1476 fps= 43 q=29.0 size= 27904kB time=00:00:47.23 bitrate=4839.6kbits/
frame= 1499 fps= 43 q=29.0 size= 28416kB time=00:00:48.00 bitrate=4849.7kbits/
frame= 1520 fps= 43 q=29.0 size= 28928kB time=00:00:48.70 bitrate=4866.1kbits/
frame= 1543 fps= 43 q=29.0 size= 29440kB time=00:00:49.46 bitrate=4875.5kbits/
frame= 1566 fps= 43 q=29.0 size= 29696kB time=00:00:50.23 bitrate=4842.8kbits/
frame= 1591 fps= 43 q=29.0 size= 30208kB time=00:00:51.06 bitrate=4845.9kbits/
frame= 1613 fps= 43 q=29.0 size= 30720kB time=00:00:51.80 bitrate=4858.3kbits/
frame= 1636 fps= 43 q=29.0 size= 31232kB time=00:00:52.56 bitrate=4867.2kbits/
frame= 1660 fps= 43 q=29.0 size= 31744kB time=00:00:53.36 bitrate=4872.8kbits/
frame= 1684 fps= 43 q=29.0 size= 32000kB time=00:00:54.16 bitrate=4839.6kbits/
frame= 1706 fps= 43 q=29.0 size= 32512kB time=00:00:54.90 bitrate=4851.3kbits/
frame= 1728 fps= 43 q=29.0 size= 33024kB time=00:00:55.63 bitrate=4862.8kbits/
frame= 1750 fps= 43 q=29.0 size= 33280kB time=00:00:56.36 bitrate=4836.7kbits/
frame= 1773 fps= 43 q=29.0 size= 33792kB time=00:00:57.13 bitrate=4845.2kbits/
frame= 1796 fps= 43 q=29.0 size= 34304kB time=00:00:57.90 bitrate=4853.5kbits/
frame= 1821 fps= 43 q=29.0 size= 34816kB time=00:00:58.73 bitrate=4856.1kbits/
frame= 1844 fps= 43 q=29.0 size= 35328kB time=00:00:59.50 bitrate=4864.0kbits/
frame= 1867 fps= 43 q=29.0 size= 35840kB time=00:01:00.26 bitrate=4871.7kbits/
frame= 1890 fps= 43 q=29.0 size= 36096kB time=00:01:01.03 bitrate=4844.9kbits/
frame= 1913 fps= 43 q=29.0 size= 36608kB time=00:01:01.80 bitrate=4852.6kbits/
frame= 1937 fps= 43 q=29.0 size= 37120kB time=00:01:02.60 bitrate=4857.6kbits/
frame= 1959 fps= 43 q=29.0 size= 37632kB time=00:01:03.33 bitrate=4867.6kbits/
frame= 1982 fps= 43 q=29.0 size= 37888kB time=00:01:04.10 bitrate=4842.1kbits/
frame= 2004 fps= 43 q=29.0 size= 38400kB time=00:01:04.83 bitrate=4852.0kbits/
frame= 2028 fps= 43 q=29.0 size= 38912kB time=00:01:05.63 bitrate=4856.8kbits/
frame= 2048 fps= 43 q=29.0 size= 39168kB time=00:01:06.30 bitrate=4839.6kbits/
frame= 2074 fps= 44 q=29.0 size= 39936kB time=00:01:07.16 bitrate=4870.8kbits/
frame= 2096 fps= 44 q=29.0 size= 40192kB time=00:01:07.90 bitrate=4849.1kbits/
frame= 2119 fps= 44 q=29.0 size= 40704kB time=00:01:08.66 bitrate=4856.0kbits/
frame= 2141 fps= 44 q=29.0 size= 41216kB time=00:01:09.40 bitrate=4865.2kbits/
frame= 2165 fps= 44 q=29.0 size= 41728kB time=00:01:10.20 bitrate=4869.5kbits/
frame= 2187 fps= 44 q=29.0 size= 41984kB time=00:01:10.93 bitrate=4848.7kbits/
frame= 2211 fps= 44 q=29.0 size= 42496kB time=00:01:11.73 bitrate=4853.1kbits/
frame= 2233 fps= 44 q=29.0 size= 43008kB time=00:01:12.46 bitrate=4861.8kbits/
frame= 2256 fps= 44 q=29.0 size= 43264kB time=00:01:13.23 bitrate=4839.6kbits/
frame= 2279 fps= 44 q=29.0 size= 43776kB time=00:01:14.00 bitrate=4846.1kbits/
frame= 2304 fps= 44 q=29.0 size= 44288kB time=00:01:14.83 bitrate=4848.2kbits/
frame= 2325 fps= 44 q=29.0 size= 44800kB time=00:01:15.53 bitrate=4858.8kbits/
frame= 2348 fps= 44 q=29.0 size= 45312kB time=00:01:16.30 bitrate=4865.0kbits/
frame= 2371 fps= 44 q=29.0 size= 45568kB time=00:01:17.06 bitrate=4843.8kbits/
frame= 2393 fps= 44 q=29.0 size= 46080kB time=00:01:17.80 bitrate=4852.0kbits/
frame= 2415 fps= 44 q=29.0 size= 46592kB time=00:01:18.53 bitrate=4860.1kbits/
frame= 2438 fps= 44 q=29.0 size= 46848kB time=00:01:19.30 bitrate=4839.6kbits/
frame= 2461 fps= 44 q=29.0 size= 47360kB time=00:01:20.06 bitrate=4845.6kbits/
frame= 2483 fps= 44 q=29.0 size= 47872kB time=00:01:20.80 bitrate=4853.6kbits/
frame= 2506 fps= 44 q=29.0 size= 48384kB time=00:01:21.56 bitrate=4859.4kbits/
frame= 2529 fps= 44 q=29.0 size= 48896kB time=00:01:22.33 bitrate=4865.1kbits/
frame= 2553 fps= 44 q=29.0 size= 49152kB time=00:01:23.13 bitrate=4843.5kbits/
frame= 2575 fps= 44 q=29.0 size= 49664kB time=00:01:23.86 bitrate=4851.1kbits/
frame= 2597 fps= 44 q=29.0 size= 50176kB time=00:01:24.60 bitrate=4858.7kbits/
frame= 2619 fps= 44 q=29.0 size= 50688kB time=00:01:25.33 bitrate=4866.0kbits/
frame= 2642 fps= 44 q=29.0 size= 50944kB time=00:01:26.10 bitrate=4847.1kbits/
frame= 2664 fps= 44 q=29.0 size= 51456kB time=00:01:26.83 bitrate=4854.4kbits/
frame= 2687 fps= 44 q=29.0 size= 51968kB time=00:01:27.60 bitrate=4859.8kbits/
frame= 2710 fps= 44 q=29.0 size= 52480kB time=00:01:28.36 bitrate=4865.1kbits/
frame= 2732 fps= 44 q=29.0 size= 52736kB time=00:01:29.10 bitrate=4848.6kbits/
frame= 2756 fps= 44 q=29.0 size= 53248kB time=00:01:29.90 bitrate=4852.1kbits/
frame= 2779 fps= 44 q=29.0 size= 53760kB time=00:01:30.66 bitrate=4857.4kbits/
frame= 2803 fps= 44 q=29.0 size= 54272kB time=00:01:31.46 bitrate=4860.7kbits/
frame= 2825 fps= 44 q=29.0 size= 54784kB time=00:01:32.20 bitrate=4867.6kbits/
frame= 2847 fps= 44 q=29.0 size= 55040kB time=00:01:32.93 bitrate=4851.7kbits/
frame= 2870 fps= 44 q=29.0 size= 55552kB time=00:01:33.70 bitrate=4856.8kbits/
frame= 2893 fps= 44 q=29.0 size= 56064kB time=00:01:34.46 bitrate=4861.8kbits/
frame= 2915 fps= 44 q=29.0 size= 56320kB time=00:01:35.20 bitrate=4846.4kbits/
frame= 2939 fps= 44 q=29.0 size= 56832kB time=00:01:36.00 bitrate=4849.7kbits/
frame= 2961 fps= 44 q=29.0 size= 57344kB time=00:01:36.73 bitrate=4856.3kbits/
frame= 2984 fps= 44 q=29.0 size= 57856kB time=00:01:37.50 bitrate=4861.1kbits/
frame= 3006 fps= 44 q=29.0 size= 58112kB time=00:01:38.23 bitrate=4846.2kbits/
frame= 3028 fps= 44 q=29.0 size= 58624kB time=00:01:38.96 bitrate=4852.6kbits/
frame= 3051 fps= 44 q=29.0 size= 59136kB time=00:01:39.73 bitrate=4857.4kbits/
frame= 3074 fps= 44 q=29.0 size= 59648kB time=00:01:40.50 bitrate=4862.1kbits/
frame= 3098 fps= 44 q=29.0 size= 60160kB time=00:01:41.30 bitrate=4865.1kbits/
frame= 3120 fps= 44 q=29.0 size= 60416kB time=00:01:42.03 bitrate=4850.6kbits/
frame= 3142 fps= 44 q=29.0 size= 60928kB time=00:01:42.76 bitrate=4856.8kbits/
frame= 3165 fps= 44 q=29.0 size= 61440kB time=00:01:43.53 bitrate=4861.4kbits/
frame= 3189 fps= 44 q=29.0 size= 61952kB time=00:01:44.33 bitrate=4864.3kbits/
frame= 3212 fps= 44 q=29.0 size= 62208kB time=00:01:45.10 bitrate=4848.8kbits/
frame= 3234 fps= 44 q=29.0 size= 62720kB time=00:01:45.83 bitrate=4854.8kbits/
frame= 3258 fps= 44 q=29.0 size= 63232kB time=00:01:46.63 bitrate=4857.7kbits/
frame= 3281 fps= 44 q=29.0 size= 63744kB time=00:01:47.40 bitrate=4862.1kbits/
frame= 3305 fps= 44 q=29.0 size= 64256kB time=00:01:48.20 bitrate=4864.9kbits/
frame= 3328 fps= 44 q=29.0 size= 64768kB time=00:01:48.96 bitrate=4869.2kbits/
frame= 3351 fps= 44 q=29.0 size= 65024kB time=00:01:49.73 bitrate=4854.3kbits/
frame= 3373 fps= 44 q=29.0 size= 65536kB time=00:01:50.46 bitrate=4860.0kbits/
frame= 3396 fps= 44 q=29.0 size= 66048kB time=00:01:51.23 bitrate=4864.2kbits/
frame= 3419 fps= 44 q=29.0 size= 66560kB time=00:01:52.00 bitrate=4868.4kbits/
frame= 3442 fps= 44 q=29.0 size= 66816kB time=00:01:52.76 bitrate=4853.9kbits/
frame= 3466 fps= 44 q=29.0 size= 67328kB time=00:01:53.56 bitrate=4856.6kbits/
frame= 3488 fps= 44 q=29.0 size= 67840kB time=00:01:54.30 bitrate=4862.2kbits/
frame= 3510 fps= 44 q=29.0 size= 68096kB time=00:01:55.03 bitrate=4849.4kbits/
frame= 3533 fps= 44 q=29.0 size= 68608kB time=00:01:55.80 bitrate=4853.5kbits/
frame= 3557 fps= 44 q=29.0 size= 69120kB time=00:01:56.60 bitrate=4856.2kbits/
frame= 3580 fps= 44 q=29.0 size= 69632kB time=00:01:57.36 bitrate=4860.2kbits/
frame= 3603 fps= 44 q=29.0 size= 70144kB time=00:01:58.13 bitrate=4864.2kbits/
frame= 3626 fps= 44 q=29.0 size= 70400kB time=00:01:58.90 bitrate=4850.4kbits/
frame= 3648 fps= 44 q=29.0 size= 70912kB time=00:01:59.63 bitrate=4855.8kbits/
frame= 3672 fps= 44 q=29.0 size= 71424kB time=00:02:00.43 bitrate=4858.3kbits/
frame= 3694 fps= 44 q=29.0 size= 71936kB time=00:02:01.16 bitrate=4863.5kbits/
frame= 3717 fps= 44 q=29.0 size= 72192kB time=00:02:01.93 bitrate=4850.2kbits/
frame= 3740 fps= 44 q=29.0 size= 72704kB time=00:02:02.70 bitrate=4854.0kbits/
frame= 3764 fps= 44 q=29.0 size= 73216kB time=00:02:03.50 bitrate=4856.6kbits/
frame= 3785 fps= 44 q=29.0 size= 73728kB time=00:02:04.20 bitrate=4863.0kbits/
frame= 3810 fps= 44 q=29.0 size= 74240kB time=00:02:05.03 bitrate=4864.1kbits/
frame= 3834 fps= 44 q=29.0 size= 74752kB time=00:02:05.83 bitrate=4866.5kbits/
frame= 3857 fps= 44 q=29.0 size= 75008kB time=00:02:06.60 bitrate=4853.6kbits/
frame= 3879 fps= 44 q=29.0 size= 75520kB time=00:02:07.33 bitrate=4858.6kbits/
frame= 3902 fps= 44 q=29.0 size= 76032kB time=00:02:08.10 bitrate=4862.2kbits/
frame= 3925 fps= 44 q=29.0 size= 76544kB time=00:02:08.86 bitrate=4865.9kbits/
frame= 3948 fps= 44 q=29.0 size= 76800kB time=00:02:09.63 bitrate=4853.3kbits/
frame= 3969 fps= 44 q=29.0 size= 77312kB time=00:02:10.33 bitrate=4859.4kbits/
frame= 3992 fps= 44 q=29.0 size= 77824kB time=00:02:11.10 bitrate=4863.0kbits/
frame= 4014 fps= 44 q=29.0 size= 78336kB time=00:02:11.83 bitrate=4867.7kbits/
frame= 4037 fps= 44 q=29.0 size= 78592kB time=00:02:12.60 bitrate=4855.4kbits/
frame= 4061 fps= 44 q=29.0 size= 79104kB time=00:02:13.40 bitrate=4857.7kbits/
frame= 4084 fps= 44 q=29.0 size= 79616kB time=00:02:14.16 bitrate=4861.2kbits/
frame= 4107 fps= 44 q=29.0 size= 80128kB time=00:02:14.93 bitrate=4864.7kbits/
frame= 4129 fps= 44 q=29.0 size= 80384kB time=00:02:15.66 bitrate=4853.9kbits/
frame= 4151 fps= 44 q=29.0 size= 80896kB time=00:02:16.40 bitrate=4858.5kbits/
frame= 4174 fps= 44 q=29.0 size= 81408kB time=00:02:17.16 bitrate=4861.9kbits/
frame= 4197 fps= 44 q=29.0 size= 81920kB time=00:02:17.93 bitrate=4865.3kbits/
frame= 4219 fps= 44 q=29.0 size= 82176kB time=00:02:18.66 bitrate=4854.7kbits/
frame= 4243 fps= 44 q=29.0 size= 82688kB time=00:02:19.46 bitrate=4856.9kbits/
frame= 4266 fps= 44 q=29.0 size= 83200kB time=00:02:20.23 bitrate=4860.3kbits/
frame= 4287 fps= 44 q=29.0 size= 83456kB time=00:02:20.93 bitrate=4851.0kbits/
frame= 4313 fps= 44 q=29.0 size= 84224kB time=00:02:21.80 bitrate=4865.7kbits/
frame= 4336 fps= 44 q=29.0 size= 84480kB time=00:02:22.56 bitrate=4854.3kbits/
frame= 4359 fps= 44 q=29.0 size= 84992kB time=00:02:23.33 bitrate=4857.6kbits/
frame= 4383 fps= 44 q=29.0 size= 85504kB time=00:02:24.13 bitrate=4859.7kbits/
frame= 4405 fps= 44 q=29.0 size= 86016kB time=00:02:24.86 bitrate=4864.1kbits/
frame= 4428 fps= 44 q=29.0 size= 86272kB time=00:02:25.63 bitrate=4852.9kbits/
frame= 4451 fps= 44 q=29.0 size= 86784kB time=00:02:26.40 bitrate=4856.1kbits/
frame= 4473 fps= 44 q=29.0 size= 87296kB time=00:02:27.13 bitrate=4860.4kbits/
frame= 4495 fps= 44 q=29.0 size= 87552kB time=00:02:27.86 bitrate=4850.5kbits/
frame= 4518 fps= 44 q=29.0 size= 88064kB time=00:02:28.63 bitrate=4853.7kbits/
frame= 4539 fps= 44 q=29.0 size= 88576kB time=00:02:29.33 bitrate=4859.0kbits/
frame= 4565 fps= 44 q=29.0 size= 89088kB time=00:02:30.20 bitrate=4858.9kbits/
frame= 4588 fps= 44 q=29.0 size= 89344kB time=00:02:30.96 bitrate=4848.1kbits/
frame= 4611 fps= 44 q=29.0 size= 89856kB time=00:02:31.73 bitrate=4851.3kbits/
frame= 4634 fps= 44 q=29.0 size= 90368kB time=00:02:32.50 bitrate=4854.4kbits/
frame= 4656 fps= 44 q=29.0 size= 90880kB time=00:02:33.23 bitrate=4858.5kbits/
frame= 4679 fps= 44 q=29.0 size= 91392kB time=00:02:34.00 bitrate=4861.6kbits/
frame= 4702 fps= 44 q=29.0 size= 91648kB time=00:02:34.76 bitrate=4851.0kbits/
frame= 4725 fps= 44 q=29.0 size= 92160kB time=00:02:35.53 bitrate=4854.1kbits/
frame= 4747 fps= 44 q=29.0 size= 92672kB time=00:02:36.26 bitrate=4858.2kbits/
frame= 4769 fps= 44 q=29.0 size= 92928kB time=00:02:37.00 bitrate=4848.8kbits/
frame= 4793 fps= 44 q=29.0 size= 93696kB time=00:02:37.80 bitrate=4864.1kbits/
frame= 4816 fps= 44 q=29.0 size= 93952kB time=00:02:38.56 bitrate=4853.8kbits/
frame= 4839 fps= 44 q=29.0 size= 94464kB time=00:02:39.33 bitrate=4856.8kbits/
frame= 4860 fps= 44 q=29.0 size= 94976kB time=00:02:40.03 bitrate=4861.8kbits/
frame= 4883 fps= 44 q=29.0 size= 95488kB time=00:02:40.80 bitrate=4864.7kbits/
frame= 4906 fps= 44 q=29.0 size= 95744kB time=00:02:41.56 bitrate=4854.6kbits/
frame= 4929 fps= 44 q=29.0 size= 96256kB time=00:02:42.33 bitrate=4857.5kbits/
frame= 4952 fps= 44 q=29.0 size= 96768kB time=00:02:43.10 bitrate=4860.4kbits/
frame= 4975 fps= 44 q=29.0 size= 97280kB time=00:02:43.86 bitrate=4863.2kbits/
frame= 4999 fps= 44 q=29.0 size= 97792kB time=00:02:44.66 bitrate=4865.1kbits/
frame= 5022 fps= 44 q=29.0 size= 98048kB time=00:02:45.43 bitrate=4855.2kbits/
frame= 5045 fps= 44 q=29.0 size= 98560kB time=00:02:46.20 bitrate=4858.0kbits/
frame= 5069 fps= 44 q=29.0 size= 99072kB time=00:02:47.00 bitrate=4859.9kbits/
frame= 5089 fps= 44 q=29.0 size= 99328kB time=00:02:47.66 bitrate=4853.1kbits/
frame= 5110 fps= 44 q=29.0 size= 99840kB time=00:02:48.36 bitrate=4857.8kbits/
frame= 5133 fps= 44 q=29.0 size= 100352kB time=00:02:49.13 bitrate=4860.6kbits/
frame= 5156 fps= 44 q=29.0 size= 100864kB time=00:02:49.90 bitrate=4863.3kbits/
frame= 5178 fps= 44 q=29.0 size= 101120kB time=00:02:50.63 bitrate=4854.7kbits/
frame= 5201 fps= 44 q=29.0 size= 101632kB time=00:02:51.40 bitrate=4857.5kbits/
frame= 5223 fps= 44 q=29.0 size= 102144kB time=00:02:52.13 bitrate=4861.1kbits/
frame= 5247 fps= 44 q=29.0 size= 102400kB time=00:02:52.93 bitrate=4850.8kbits/
frame= 5270 fps= 44 q=29.0 size= 102912kB time=00:02:53.70 bitrate=4853.5kbits/
frame= 5294 fps= 44 q=29.0 size= 103424kB time=00:02:54.50 bitrate=4855.3kbits/
frame= 5317 fps= 44 q=29.0 size= 103936kB time=00:02:55.26 bitrate=4858.0kbits/
frame= 5341 fps= 44 q=29.0 size= 104448kB time=00:02:56.06 bitrate=4859.7kbits/
frame= 5363 fps= 44 q=29.0 size= 104704kB time=00:02:56.80 bitrate=4851.4kbits/
frame= 5387 fps= 44 q=29.0 size= 105216kB time=00:02:57.60 bitrate=4853.2kbits/
frame= 5410 fps= 44 q=29.0 size= 105728kB time=00:02:58.36 bitrate=4855.9kbits/
frame= 5433 fps= 44 q=29.0 size= 105984kB time=00:02:59.13 bitrate=4846.8kbits/
frame= 5457 fps= 44 q=29.0 size= 106496kB time=00:02:59.93 bitrate=4848.5kbits/
frame= 5479 fps= 44 q=29.0 size= 107008kB time=00:03:00.66 bitrate=4852.1kbits/
frame= 5502 fps= 44 q=29.0 size= 107520kB time=00:03:01.43 bitrate=4854.7kbits/
frame= 5524 fps= 44 q=26.0 size= 107776kB time=00:03:02.16 bitrate=4846.7kbits/
frame= 5550 fps= 44 q=29.0 size= 108288kB time=00:03:03.03 bitrate=4846.6kbits/
frame= 5571 fps= 44 q=29.0 size= 108800kB time=00:03:03.73 bitrate=4851.0kbits/
frame= 5594 fps= 44 q=29.0 size= 109312kB time=00:03:04.50 bitrate=4853.6kbits/
frame= 5616 fps= 44 q=29.0 size= 109568kB time=00:03:05.23 bitrate=4845.7kbits/
frame= 5639 fps= 44 q=29.0 size= 110080kB time=00:03:06.00 bitrate=4848.3kbits/
frame= 5663 fps= 44 q=29.0 size= 110592kB time=00:03:06.80 bitrate=4849.9kbits/
frame= 5686 fps= 44 q=29.0 size= 111104kB time=00:03:07.56 bitrate=4852.5kbits/
frame= 5697 fps= 44 q=-1.0 Lsize= 112523kB time=00:03:09.80 bitrate=4856.6kbits
/s speed=1.46x
video:112497kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxin
g overhead: 0.022979%
[libx264 @ 0000000004107d80] frame I:32 Avg QP:23.66 size: 43890
[libx264 @ 0000000004107d80] frame P:5534 Avg QP:26.46 size: 20105
[libx264 @ 0000000004107d80] frame B:131 Avg QP:26.63 size: 19331
[libx264 @ 0000000004107d80] consecutive B-frames: 96.8% 0.3% 0.2% 2.7%
[libx264 @ 0000000004107d80] mb I I16..4: 32.4% 44.4% 23.2%
[libx264 @ 0000000004107d80] mb P I16..4: 3.0% 30.0% 1.6% P16..4: 46.2% 14.2
% 4.7% 0.0% 0.0% skip: 0.2%
[libx264 @ 0000000004107d80] mb B I16..4: 1.3% 6.6% 0.3% B16..8: 54.1% 20.5
% 2.2% direct:14.3% skip: 0.8% L0:54.3% L1:43.7% BI: 2.1%
[libx264 @ 0000000004107d80] 8x8 transform intra:86.0% inter:63.3%
[libx264 @ 0000000004107d80] coded y,uvDC,uvAC intra: 82.2% 0.0% 0.0% inter: 60.
2% 0.0% 0.0%
[libx264 @ 0000000004107d80] i16 v,h,dc,p: 14% 17% 52% 17%
[libx264 @ 0000000004107d80] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 11% 13% 26% 8% 9%
8% 9% 7% 9%
[libx264 @ 0000000004107d80] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 21% 19% 30% 7% 6%
5% 5% 4% 4%
[libx264 @ 0000000004107d80] i8c dc,h,v,p: 100% 0% 0% 0%
[libx264 @ 0000000004107d80] Weighted P-Frames: Y:27.3% UV:0.0%
[libx264 @ 0000000004107d80] ref P L0: 39.5% 12.6% 23.5% 19.6% 4.9%
[libx264 @ 0000000004107d80] ref B L0: 78.2% 18.3% 3.6%
[libx264 @ 0000000004107d80] ref B L1: 85.6% 14.4%
[libx264 @ 0000000004107d80] kb/s:4852.91
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 08:50:29.166 NotebookApp] Saving file at /pipeline.ipynb
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 08:52:29.167 NotebookApp] Saving file at /pipeline.ipynb
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 08:54:29.173 NotebookApp] Saving file at /pipeline.ipynb
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 08:56:29.179 NotebookApp] Saving file at /pipeline.ipynb
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 08:58:29.215 NotebookApp] Saving file at /pipeline.ipynb
[I 09:00:29.189 NotebookApp] Saving file at /pipeline.ipynb
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 09:02:29.198 NotebookApp] Saving file at /pipeline.ipynb
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 09:04:29.199 NotebookApp] Saving file at /pipeline.ipynb
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 09:10:29.211 NotebookApp] Saving file at /pipeline.ipynb
[I 09:12:29.399 NotebookApp] Saving file at /pipeline.ipynb
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
X:\minian-dev\minian\cnmf.py:769: RuntimeWarning: problem solved sub-optimally
warnings.warn("problem solved sub-optimally", RuntimeWarning)
[I 09:16:29.630 NotebookApp] Saving file at /pipeline.ipynb
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
cannot use cuda accelerate
[IPKernelApp] WARNING | No such comm: hv-extension-comm
[I 09:18:29.237 NotebookApp] Saving file at /pipeline.ipynb
is there a way to enable cuda? I have a 2070 card installed and tensorflow gpu working
thanks!
richardson
obtaining some hand-scoring data to validate / compare the performance of minian.
Can be done in various ways providing various level of information:
Hi
I was wondering if there is any way to import a series of tiff files instead of avi files as raw input data?
Implement some selections tools in CNMF viewer and save as boolean masks.
When batch, before going into CNMF, it would be helpful if can check the quality of motion correction! Plot the shifts and post-alignment video on one page. (with buttons :) 💯 )
Plots: 1, Height/Width Shift Plot
2, Heat-map Shift Plot
Video: The Result of Motion Correction
Thanks Phil!! :))
This error appears during spatial update on the perf branch:
bool
to np.generic
is deprecated. In future, it will be treated as np.bool_ == np.dtype(bool).type
.ValueError Traceback (most recent call last)
in
~/GitHub/minian/minian/cnmf.py in update_spatial(Y, A, b, C, f, sn, dl_wnd, sparse_penal, update_background, normalize, size_thres, in_memory)
161 )
162 with parallel_backend("dask"):
--> 163 A_new = save_minian(
164 A_new.rename("A_new"),
165 intpath,
~/GitHub/minian/minian/utilities.py in save_minian(var, dpath, meta_dict, overwrite, chunks, mem_limit)
228 except FileNotFoundError:
229 pass
--> 230 ds.to_zarr(fp, mode=md)
231 if chunks is not None:
232 chunks = {d: var.sizes[d] if v <= 0 else v for d, v in chunks.items()}
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/core/dataset.py in to_zarr(self, store, mode, synchronizer, group, encoding, compute, consolidated, append_dim)
1614 from ..backends.api import to_zarr
1615
-> 1616 return to_zarr(
1617 self,
1618 store=store,
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/backends/api.py in to_zarr(dataset, store, mode, synchronizer, group, encoding, compute, consolidated, append_dim)
1342 # TODO: figure out how to properly handle unlimited_dims
1343 dump_to_store(dataset, zstore, writer, encoding=encoding)
-> 1344 writes = writer.sync(compute=compute)
1345
1346 if compute:
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/backends/common.py in sync(self, compute)
196 # targets = [dask.delayed(t) for t in self.targets]
197
--> 198 delayed_store = da.store(
199 self.sources,
200 self.targets,
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/array/core.py in store(sources, targets, lock, regions, compute, return_stored, **kwargs)
912
913 if compute:
--> 914 result.compute(**kwargs)
915 return None
916 else:
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/base.py in compute(self, **kwargs)
164 dask.base.compute
165 """
--> 166 (result,) = compute(self, traverse=False, **kwargs)
167 return result
168
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/base.py in compute(*args, **kwargs)
435 keys = [x.dask_keys() for x in collections]
436 postcomputes = [x.dask_postcompute() for x in collections]
--> 437 results = schedule(dsk, keys, **kwargs)
438 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
439
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in get(self, dsk, keys, restrictions, loose_restrictions, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
2593 should_rejoin = False
2594 try:
-> 2595 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
2596 finally:
2597 for f in futures.values():
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in gather(self, futures, errors, direct, asynchronous)
1885 else:
1886 local_worker = None
-> 1887 return self.sync(
1888 self._gather,
1889 futures,
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
777 return future
778 else:
--> 779 return sync(
780 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
781 )
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
346 if error[0]:
347 typ, exc, tb = error[0]
--> 348 raise exc.with_traceback(tb)
349 else:
350 return result[0]
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/utils.py in f()
330 if callback_timeout is not None:
331 future = asyncio.wait_for(future, callback_timeout)
--> 332 result[0] = yield future
333 except Exception as exc:
334 error[0] = sys.exc_info()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/tornado/gen.py in run(self)
733
734 try:
--> 735 value = future.result()
736 except Exception:
737 exc_info = sys.exc_info()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
1750 exc = CancelledError(key)
1751 else:
-> 1752 raise exception.with_traceback(traceback)
1753 raise exc
1754 if errors == "skip":
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/optimization.py in call()
980 if not len(args) == len(self.inkeys):
981 raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args)))
--> 982 return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
983
984 def reduce(self):
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/core.py in get()
147 for key in toposort(dsk):
148 task = dsk[key]
--> 149 result = _execute_task(task, cache)
150 cache[key] = result
151 result = _execute_task(out, cache)
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/core.py in _execute_task()
117 func, args = arg[0], arg[1:]
118 args2 = [_execute_task(a, cache) for a in args]
--> 119 return func(*args2)
120 elif not ishashable(arg):
121 return arg
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/utils.py in apply()
27 def apply(func, args, kwargs=None):
28 if kwargs:
---> 29 return func(*args, **kwargs)
30 else:
31 return func(*args)
~/GitHub/minian/minian/cnmf.py in update_spatial_block()
234 data_ls = []
235 for h, w in zip(*sub.any(axis=-1).nonzero()):
--> 236 res = update_spatial_perpx(y[h, w, :], alpha[h, w], sub[h, w, :], C_store, f)
237 crd = res.coords
238 crd = np.concatenate([np.full_like(crd, h), np.full_like(crd, w), crd], axis=0)
~/GitHub/minian/minian/cnmf.py in update_spatial_perpx()
217 C = C_store[idx, :].T
218 if (f is not None) and sub[-1]:
--> 219 C = np.concatenate([C, f.reshape((-1, 1))], axis=1)
220 idx = np.concatenate([idx, np.array(len(sub) - 1).reshape(-1)])
221 clf = LassoLars(alpha=alpha, positive=True)
<array_function internals> in concatenate()
ValueError: all the input array dimensions for the concatenation axis must match exactly, but along dimension 0, the array at index 0 has size 8970 and the array at index 1 has size 35743
Hello,
the main branch seems to have compatibility issues between some of the libraries on my computer. I found similar issues filed (#48 (comment), #45 (comment)) and decided to try both the latest dev version and the 12dae24.
I have two datasets, Set 1 (17GB) and Set 2 (8GB). I ran the latest dev version (with .compute()s) with Set 1, which decided to save all the video chunks into the data folder while running the cell that saves the motion-corrected video. When I commented out this cell, it reached the generate_videos function, which populated the data folder with hundreds of video chunks in the writing_videos step. I have not tried to run Set 2 with this version.
The 12dae24 version (with .compute()s) managed to run the whole Set 2 without any issue. However, it failed to run the Set 1 data, with a seemingly identical issue as the latest dev version. I tried running it after resetting the kernel to face the same issue. The dpath is exactly the same except the number 1 and 2.
The following is the output from the generate_video cell in the visualization section of 12dae24, and a screenshot of the data folder.
generating traces
[########################################] | 100% Completed | 41.5s
cannot find background term
[########################################] | 100% Completed | 7min 25.2s
writing videos
[########################################] | 100% Completed | 2min 27.1s
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
<timed eval> in <module>
~\Documents\GitHub\minian-old\minian\visualization.py in generate_videos(minian, varr, vpath, vname, scale)
1244 res_norm = Y_norm - AC_norm
1245 print("writing videos")
-> 1246 path_org = write_video(org_norm, vpath=vpath)
1247 path_Y = write_video(Y_norm, vpath=vpath)
1248 path_AC = write_video(AC_norm, vpath=vpath)
~\Documents\GitHub\minian-old\minian\visualization.py in write_video(arr, vname, vpath)
1203 paths = dask.compute(paths)[0]
1204 streams = [ffmpeg.input(p) for p in paths]
-> 1205 (ffmpeg.concat(*streams)
1206 .output(fname)
1207 .run(overwrite_output=True))
~\.conda\envs\minian_old\lib\site-packages\ffmpeg\_run.py in run(stream_spec, cmd, capture_stdout, capture_stderr, input, quiet, overwrite_output)
311 Returns: (out, err) tuple containing captured stdout and stderr data.
312 """
--> 313 process = run_async(
314 stream_spec,
315 cmd,
~\.conda\envs\minian_old\lib\site-packages\ffmpeg\_run.py in run_async(stream_spec, cmd, pipe_stdin, pipe_stdout, pipe_stderr, quiet, overwrite_output)
282 stdout_stream = subprocess.PIPE if pipe_stdout or quiet else None
283 stderr_stream = subprocess.PIPE if pipe_stderr or quiet else None
--> 284 return subprocess.Popen(
285 args, stdin=stdin_stream, stdout=stdout_stream, stderr=stderr_stream
286 )
~\.conda\envs\minian_old\lib\subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
852 encoding=encoding, errors=errors)
853
--> 854 self._execute_child(args, executable, preexec_fn, close_fds,
855 pass_fds, cwd, env,
856 startupinfo, creationflags, shell,
~\.conda\envs\minian_old\lib\subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session)
1305 # Start the process
1306 try:
-> 1307 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
1308 # no special security
1309 None, None,
FileNotFoundError: [WinError 206] The filename or extension is too long
When the npart
parameter in estimate_shifts
is too low (<18 for a 20-30 min movie), save_minian
throws this error:
TypeError Traceback (most recent call last)
in
~/GitHub/minian/minian/utilities.py in save_minian(var, dpath, meta_dict, overwrite, chunks, mem_limit)
253 except FileNotFoundError:
254 pass
--> 255 ds.to_zarr(fp, mode=md)
256 if chunks is not None:
257 chunks = {d: var.sizes[d] if v <= 0 else v for d, v in chunks.items()}
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/core/dataset.py in to_zarr(self, store, mode, synchronizer, group, encoding, compute, consolidated, append_dim)
1614 from ..backends.api import to_zarr
1615
-> 1616 return to_zarr(
1617 self,
1618 store=store,
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/backends/api.py in to_zarr(dataset, store, mode, synchronizer, group, encoding, compute, consolidated, append_dim)
1342 # TODO: figure out how to properly handle unlimited_dims
1343 dump_to_store(dataset, zstore, writer, encoding=encoding)
-> 1344 writes = writer.sync(compute=compute)
1345
1346 if compute:
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/backends/common.py in sync(self, compute)
196 # targets = [dask.delayed(t) for t in self.targets]
197
--> 198 delayed_store = da.store(
199 self.sources,
200 self.targets,
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/array/core.py in store(sources, targets, lock, regions, compute, return_stored, **kwargs)
912
913 if compute:
--> 914 result.compute(**kwargs)
915 return None
916 else:
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/base.py in compute(self, **kwargs)
164 dask.base.compute
165 """
--> 166 (result,) = compute(self, traverse=False, **kwargs)
167 return result
168
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/base.py in compute(*args, **kwargs)
435 keys = [x.dask_keys() for x in collections]
436 postcomputes = [x.dask_postcompute() for x in collections]
--> 437 results = schedule(dsk, keys, **kwargs)
438 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
439
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in get(self, dsk, keys, restrictions, loose_restrictions, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
2593 should_rejoin = False
2594 try:
-> 2595 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
2596 finally:
2597 for f in futures.values():
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in gather(self, futures, errors, direct, asynchronous)
1885 else:
1886 local_worker = None
-> 1887 return self.sync(
1888 self._gather,
1889 futures,
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
777 return future
778 else:
--> 779 return sync(
780 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
781 )
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
346 if error[0]:
347 typ, exc, tb = error[0]
--> 348 raise exc.with_traceback(tb)
349 else:
350 return result[0]
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/utils.py in f()
330 if callback_timeout is not None:
331 future = asyncio.wait_for(future, callback_timeout)
--> 332 result[0] = yield future
333 except Exception as exc:
334 error[0] = sys.exc_info()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/tornado/gen.py in run(self)
733
734 try:
--> 735 value = future.result()
736 except Exception:
737 exc_info = sys.exc_info()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
1750 exc = CancelledError(key)
1751 else:
-> 1752 raise exception.with_traceback(traceback)
1753 raise exc
1754 if errors == "skip":
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/optimization.py in call()
980 if not len(args) == len(self.inkeys):
981 raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args)))
--> 982 return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
983
984 def reduce(self):
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/core.py in get()
147 for key in toposort(dsk):
148 task = dsk[key]
--> 149 result = _execute_task(task, cache)
150 cache[key] = result
151 result = _execute_task(out, cache)
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/core.py in _execute_task()
117 func, args = arg[0], arg[1:]
118 args2 = [_execute_task(a, cache) for a in args]
--> 119 return func(*args2)
120 elif not ishashable(arg):
121 return arg
~/GitHub/minian/minian/motion_correction.py in est_sh_part()
116 else:
117 continue
--> 118 sh_add = match_func(fm, temp, max_sh, local)
119 for j in sh_idx:
120 sh_ls[j] = sh_ls[j] + sh_add.reshape((1, -1))
~/GitHub/minian/minian/motion_correction.py in match_temp()
142 [array]: array (x,y) of the shift (match)
143 """
--> 144 src = np.pad(src, max_sh)
145 cor = cv2.matchTemplate(
146 src.astype(np.float32), dst.astype(np.float32), cv2.TM_CCOEFF_NORMED
<array_function internals> in pad()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/array/core.py in array_function()
1348 if da_func is func:
1349 return handle_nonmatching_names(func, args, kwargs)
-> 1350 return da_func(*args, **kwargs)
1351
1352 @Property
TypeError: pad() missing 1 required positional argument: 'mode'
Got this error during the PNR refinement step on the perf
branch.
`ValueError Traceback (most recent call last)
in
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/core/dataarray.py in compute(self, **kwargs)
825 """
826 new = self.copy(deep=False)
--> 827 return new.load(**kwargs)
828
829 def persist(self, **kwargs) -> "DataArray":
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/core/dataarray.py in load(self, **kwargs)
799 dask.array.compute
800 """
--> 801 ds = self._to_temp_dataset().load(**kwargs)
802 new = self._from_temp_dataset(ds)
803 self._variable = new._variable
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/xarray/core/dataset.py in load(self, **kwargs)
649
650 # evaluate all the dask arrays simultaneously
--> 651 evaluated_data = da.compute(*lazy_data.values(), **kwargs)
652
653 for k, data in zip(lazy_data, evaluated_data):
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/base.py in compute(*args, **kwargs)
435 keys = [x.dask_keys() for x in collections]
436 postcomputes = [x.dask_postcompute() for x in collections]
--> 437 results = schedule(dsk, keys, **kwargs)
438 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
439
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in get(self, dsk, keys, restrictions, loose_restrictions, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
2593 should_rejoin = False
2594 try:
-> 2595 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
2596 finally:
2597 for f in futures.values():
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in gather(self, futures, errors, direct, asynchronous)
1885 else:
1886 local_worker = None
-> 1887 return self.sync(
1888 self._gather,
1889 futures,
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
777 return future
778 else:
--> 779 return sync(
780 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
781 )
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
346 if error[0]:
347 typ, exc, tb = error[0]
--> 348 raise exc.with_traceback(tb)
349 else:
350 return result[0]
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/utils.py in f()
330 if callback_timeout is not None:
331 future = asyncio.wait_for(future, callback_timeout)
--> 332 result[0] = yield future
333 except Exception as exc:
334 error[0] = sys.exc_info()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/tornado/gen.py in run(self)
733
734 try:
--> 735 value = future.result()
736 except Exception:
737 exc_info = sys.exc_info()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
1750 exc = CancelledError(key)
1751 else:
-> 1752 raise exception.with_traceback(traceback)
1753 raise exc
1754 if errors == "skip":
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/optimization.py in call()
980 if not len(args) == len(self.inkeys):
981 raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args)))
--> 982 return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
983
984 def reduce(self):
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/core.py in get()
147 for key in toposort(dsk):
148 task = dsk[key]
--> 149 result = _execute_task(task, cache)
150 cache[key] = result
151 result = _execute_task(out, cache)
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/dask/core.py in _execute_task()
117 func, args = arg[0], arg[1:]
118 args2 = [_execute_task(a, cache) for a in args]
--> 119 return func(*args2)
120 elif not ishashable(arg):
121 return arg
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/numpy/lib/function_base.py in call()
2089 vargs.extend([kwargs[_n] for _n in names])
2090
-> 2091 return self._vectorize_call(func=func, args=vargs)
2092
2093 def _get_ufunc_and_otypes(self, func, args):
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/numpy/lib/function_base.py in _vectorize_call()
2155 """Vectorized call to func
over positional args
."""
2156 if self.signature is not None:
-> 2157 res = self._vectorize_call_with_signature(func, args)
2158 elif not args:
2159 res = func()
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/numpy/lib/function_base.py in _vectorize_call_with_signature()
2210 if outputs is None:
2211 for result, core_dims in zip(results, output_core_dims):
-> 2212 _update_dim_sizes(dim_sizes, result, core_dims)
2213
2214 if otypes is None:
/opt/miniconda3/envs/minian-perf/lib/python3.8/site-packages/numpy/lib/function_base.py in _update_dim_sizes()
1826 if dim in dim_sizes:
1827 if size != dim_sizes[dim]:
-> 1828 raise ValueError(
1829 'inconsistent size for core dimension %r: %r vs %r'
1830 % (dim, size, dim_sizes[dim]))
ValueError: inconsistent size for core dimension 'dim0': 35980 vs 35981`
Hi Dong,
While analyzing my data using minian, I ran into a problem at a certain step of the analysis, that takes a lot of time than I expected. It was during 'first spatial update' in CNMF.
Specification of my computer is :
OS : Linux chaeyoungkim-Z68X-UD3H-B3 4.4.0-21-generic #37-Ubuntu SMP Mon Apr 18 18:33:37 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
Hardware :
-4 processors (Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz)
-Memory :
total used free shared buff/cache available
Mem: 15964 11804 707 127 3452 3900
Swap: 9585 4820 4765
Packages:
_libgcc_mutex 0.1 main
ansiwrap 0.8.4 py_0 conda-forge
asciitree 0.3.3 py_2 conda-forge
attrs 19.3.0 py_0 conda-forge
av 6.2.0 py37h866369f_1 conda-forge
backcall 0.1.0 py_0 conda-forge
backports 1.0 py_2 conda-forge
backports.tempfile 1.0 py_0 conda-forge
backports.weakref 1.0.post1 py37_1000 conda-forge
bleach 3.1.0 py_0 conda-forge
bokeh 1.4.0 py37_0 conda-forge
bottleneck 1.3.0 py37hc1659b7_0 conda-forge
bzip2 1.0.8 h516909a_1 conda-forge
ca-certificates 2019.9.11 hecc5488_0 conda-forge
cairo 1.16.0 hfb77d84_1002 conda-forge
certifi 2019.9.11 py37_0 conda-forge
cffi 1.13.2 py37h8022711_0 conda-forge
cftime 1.0.4.2 py37hc1659b7_0 conda-forge
chardet 3.0.4 py37_1003 conda-forge
click 7.0 py_0 conda-forge
cloudpickle 1.2.2 py_0 conda-forge
colorcet 2.0.1 py_0 conda-forge
cryptography 2.8 py37h72c5cf5_0 conda-forge
curl 7.65.3 hf8cf82a_0 conda-forge
cvxpy 1.0.25 py37he1b5a44_2 conda-forge
cvxpy-base 1.0.25 py37he1b5a44_2 conda-forge
cycler 0.10.0 py_2 conda-forge
cytoolz 0.10.1 py37h516909a_0 conda-forge
dask 1.2.2 py_3 conda-forge
dask-core 1.2.2 py_0 conda-forge
datashader 0.8.0 py_0 conda-forge
datashape 0.5.4 py_1 conda-forge
dbus 1.13.6 he372182_0 conda-forge
decorator 4.4.1 py_0 conda-forge
defusedxml 0.6.0 py_0 conda-forge
dill 0.3.1.1 py37_0 conda-forge
distributed 1.28.1 py37_0 conda-forge
ecos 2.0.7.post1 py37hc1659b7_0 conda-forge
entrypoints 0.3 py37_1000 conda-forge
expat 2.2.5 he1b5a44_1004 conda-forge
fastcache 1.1.0 py37h516909a_0 conda-forge
fasteners 0.14.1 py_3 conda-forge
ffmpeg 4.1.3 h167e202_0 conda-forge
ffmpeg-python 0.2.0 pypi_0 pypi
fftw 3.3.8 nompi_h7f3a6c3_1110 conda-forge
fontconfig 2.13.1 h86ecdb6_1001 conda-forge
freetype 2.10.0 he983fc9_1 conda-forge
future 0.18.2 py37_0 conda-forge
gettext 0.19.8.1 hc5be6a0_1002 conda-forge
giflib 5.1.9 h516909a_0 conda-forge
glib 2.58.3 py37h6f030ca_1002 conda-forge
gmp 6.1.2 hf484d3e_1000 conda-forge
gnutls 3.6.5 hd3a4fd2_1002 conda-forge
graphite2 1.3.13 hf484d3e_1000 conda-forge
gst-plugins-base 1.14.5 h0935bb2_0 conda-forge
gstreamer 1.14.5 h36ae1b5_0 conda-forge
harfbuzz 2.4.0 h9f30f68_3 conda-forge
hdf4 4.2.13 hf30be14_1003 conda-forge
hdf5 1.10.5 nompi_h3c11f04_1104 conda-forge
heapdict 1.0.1 py_0 conda-forge
holoviews 1.12.6 py_0 conda-forge
icu 64.2 he1b5a44_1 conda-forge
idna 2.8 py37_1000 conda-forge
imageio 2.6.1 py37_0 conda-forge
importlib_metadata 0.23 py37_0 conda-forge
ipykernel 5.1.3 py37h5ca1d4c_0 conda-forge
ipython 7.9.0 py37h5ca1d4c_0 conda-forge
ipython_genutils 0.2.0 py_1 conda-forge
ipywidgets 7.5.1 py_0 conda-forge
jasper 1.900.1 h07fcdf6_1006 conda-forge
jedi 0.15.1 py37_0 conda-forge
jinja2 2.10.3 py_0 conda-forge
joblib 0.14.0 py_0 conda-forge
jpeg 9c h14c3975_1001 conda-forge
jsonschema 3.1.1 py37_0 conda-forge
jupyter 1.0.0 py_2 conda-forge
jupyter_client 5.3.3 py37_1 conda-forge
jupyter_console 6.0.0 py_0 conda-forge
jupyter_core 4.5.0 py_0 conda-forge
kiwisolver 1.1.0 py37hc9558a2_0 conda-forge
krb5 1.16.3 h05b26f9_1001 conda-forge
lame 3.100 h14c3975_1001 conda-forge
libblas 3.8.0 14_openblas conda-forge
libcblas 3.8.0 14_openblas conda-forge
libclang 9.0.0 default_hde54327_4 conda-forge
libcurl 7.65.3 hda55be3_0 conda-forge
libedit 3.1.20170329 hf8c457e_1001 conda-forge
libffi 3.2.1 he1b5a44_1006 conda-forge
libgcc-ng 9.1.0 hdf63c60_0
libgfortran-ng 7.3.0 hdf63c60_2 conda-forge
libiconv 1.15 h516909a_1005 conda-forge
liblapack 3.8.0 14_openblas conda-forge
liblapacke 3.8.0 14_openblas conda-forge
libllvm8 8.0.1 hc9558a2_0 conda-forge
libllvm9 9.0.0 hc9558a2_3 conda-forge
libnetcdf 4.7.1 nompi_h94020b1_101 conda-forge
libopenblas 0.3.7 h6e990d7_3 conda-forge
libpng 1.6.37 hed695b0_0 conda-forge
libsodium 1.0.17 h516909a_0 conda-forge
libssh2 1.8.2 h22169c7_2 conda-forge
libstdcxx-ng 9.1.0 hdf63c60_0
libtiff 4.1.0 hfc65ed5_0 conda-forge
libuuid 2.32.1 h14c3975_1000 conda-forge
libuv 1.33.1 h516909a_0 conda-forge
libwebp 1.0.2 h576950b_1 conda-forge
libxcb 1.13 h14c3975_1002 conda-forge
libxkbcommon 0.9.1 hebb1f50_0 conda-forge
libxml2 2.9.10 hee79883_0 conda-forge
llvmlite 0.30.0 py37h8b12597_1 conda-forge
locket 0.2.0 py_2 conda-forge
lz4-c 1.8.3 he1b5a44_1001 conda-forge
markdown 3.1.1 py_0 conda-forge
markupsafe 1.1.1 py37h516909a_0 conda-forge
matplotlib 3.1.2 py37_0 conda-forge
matplotlib-base 3.1.2 py37h250f245_0 conda-forge
medpy 0.4.0 pypi_0 pypi
mistune 0.8.4 py37h516909a_1000 conda-forge
monotonic 1.5 py_0 conda-forge
more-itertools 7.2.0 py_0 conda-forge
msgpack-python 0.6.2 py37hc9558a2_0 conda-forge
multipledispatch 0.6.0 py_0 conda-forge
multiprocess 0.70.9 py37h516909a_0 conda-forge
natsort 6.1.0 py_0 conda-forge
nbconvert 5.6.1 py37_0 conda-forge
nbformat 4.4.0 py_1 conda-forge
ncurses 6.1 hf484d3e_1002 conda-forge
netcdf4 1.5.3 py37hf3213d6_0 conda-forge
nettle 3.4.1 h1bed415_1002 conda-forge
networkx 2.4 py_0 conda-forge
nodejs 12.13.0 h10a4023_0 conda-forge
notebook 6.0.1 py37_0 conda-forge
nspr 4.23 he1b5a44_0 conda-forge
nss 3.47 he751ad9_0 conda-forge
numba 0.46.0 py37hb3f55d8_1 conda-forge
numcodecs 0.6.4 py37he1b5a44_0 conda-forge
numpy 1.17.3 py37h95a1406_0 conda-forge
olefile 0.46 py_0 conda-forge
opencv 4.1.1 py37h0cc45ee_2 conda-forge
openh264 1.8.0 hdbcaa40_1000 conda-forge
openssl 1.1.1d h516909a_0 conda-forge
osqp 0.6.1 py37hb3f55d8_1 conda-forge
packaging 19.2 py_0 conda-forge
pandas 0.25.3 py37hb3f55d8_0 conda-forge
pandoc 2.7.3 0 conda-forge
pandocfilters 1.4.2 py_1 conda-forge
panel 0.6.4 0 conda-forge
papermill 1.2.1 py37_0 conda-forge
param 1.9.2 py_0 conda-forge
parso 0.5.1 py_0 conda-forge
partd 1.0.0 py_0 conda-forge
patsy 0.5.1 py_0 conda-forge
pcre 8.43 he1b5a44_0 conda-forge
pexpect 4.7.0 py37_0 conda-forge
pickleshare 0.7.5 py37_1000 conda-forge
pillow 6.2.1 py37h6b7be26_0 conda-forge
pip 19.3.1 py37_0 conda-forge
pixman 0.38.0 h516909a_1003 conda-forge
prometheus_client 0.7.1 py_0 conda-forge
prompt_toolkit 2.0.10 py_0 conda-forge
psutil 5.6.5 py37h516909a_0 conda-forge
pthread-stubs 0.4 h14c3975_1001 conda-forge
ptyprocess 0.6.0 py_1001 conda-forge
pycparser 2.19 py37_1 conda-forge
pyct 0.4.6 py_0 conda-forge
pyct-core 0.4.6 py_0 conda-forge
pyfftw 0.11.1 py37h240ffea_1002 conda-forge
pygments 2.4.2 py_0 conda-forge
pyopenssl 19.0.0 py37_0 conda-forge
pyparsing 2.4.5 py_0 conda-forge
pyqt 5.12.3 py37hcca6a23_1 conda-forge
pyqt5-sip 4.19.18 pypi_0 pypi
pyqtwebengine 5.12.1 pypi_0 pypi
pyrsistent 0.15.5 py37h516909a_0 conda-forge
pysocks 1.7.1 py37_0 conda-forge
python 3.7.3 h33d41f4_1 conda-forge
python-dateutil 2.8.1 py_0 conda-forge
pytz 2019.3 py_0 conda-forge
pyviz_comms 0.7.2 py_0 conda-forge
pywavelets 1.1.1 py37hc1659b7_0 conda-forge
pyyaml 5.1.2 py37h516909a_0 conda-forge
pyzmq 18.1.0 py37h1768529_0 conda-forge
qt 5.12.5 hd8c4c69_1 conda-forge
qtconsole 4.5.5 py_0 conda-forge
readline 8.0 hf8c457e_0 conda-forge
requests 2.22.0 py37_1 conda-forge
scikit-image 0.16.2 py37hb3f55d8_0 conda-forge
scikit-learn 0.21.3 py37hcdab131_0 conda-forge
scipy 1.3.2 py37h921218d_0 conda-forge
scs 2.1.1.2 py37h4ff444d_3 conda-forge
send2trash 1.5.0 py_0 conda-forge
setuptools 41.6.0 py37_1 conda-forge
simpleitk 1.2.3 pypi_0 pypi
six 1.13.0 py37_0 conda-forge
sk-video 1.1.10 pypi_0 pypi
sortedcontainers 2.1.0 py_0 conda-forge
sqlite 3.30.1 hcee41ef_0 conda-forge
statsmodels 0.10.1 py37hc1659b7_2 conda-forge
tblib 1.4.0 py_0 conda-forge
tenacity 6.0.0 py37_0 conda-forge
terminado 0.8.3 py37_0 conda-forge
testpath 0.4.4 py_0 conda-forge
textwrap3 0.9.2 py_0 conda-forge
tifffile 0.15.1 py37hc1659b7_1001 conda-forge
tk 8.6.9 hed695b0_1003 conda-forge
toolz 0.10.0 py_0 conda-forge
tornado 6.0.3 py37h516909a_0 conda-forge
tqdm 4.38.0 py_0 conda-forge
traitlets 4.3.3 py37_0 conda-forge
urllib3 1.25.7 py37_0 conda-forge
wcwidth 0.1.7 py_1 conda-forge
webencodings 0.5.1 py_1 conda-forge
wheel 0.33.6 py37_0 conda-forge
widgetsnbextension 3.5.1 py37_0 conda-forge
x264 1!152.20180806 h14c3975_0 conda-forge
xarray 0.14.0 py_0 conda-forge
xorg-kbproto 1.0.7 h14c3975_1002 conda-forge
xorg-libice 1.0.10 h516909a_0 conda-forge
xorg-libsm 1.2.3 h84519dc_1000 conda-forge
xorg-libx11 1.6.9 h516909a_0 conda-forge
xorg-libxau 1.0.9 h14c3975_0 conda-forge
xorg-libxdmcp 1.1.3 h516909a_0 conda-forge
xorg-libxext 1.3.4 h516909a_0 conda-forge
xorg-libxrender 0.9.10 h516909a_1002 conda-forge
xorg-renderproto 0.11.1 h14c3975_1002 conda-forge
xorg-xextproto 7.3.0 h14c3975_1002 conda-forge
xorg-xproto 7.0.31 h14c3975_1007 conda-forge
xz 5.2.4 h14c3975_1001 conda-forge
yaml 0.1.7 h14c3975_1001 conda-forge
zarr 2.3.2 py37_0 conda-forge
zeromq 4.3.2 he1b5a44_2 conda-forge
zict 1.0.0 py_0 conda-forge
zipp 0.6.0 py_0 conda-forge
zlib 1.2.11 h516909a_1006 conda-forge
zstd 1.4.3 h3b9ef0a_0 conda-forge
Here I attach a screenshot of my monitor.
Thank you so much!
Just downloaded a more recent version of minian, it seems that the "compute" argument has been removed from the VArrayViewer, but the pipelines still pass "compute=True" as an argument
%%time
seeds = seeds_init(Y, **param_seeds_init)
constructing chunks
TypeError Traceback (most recent call last)
~\Anaconda3\envs\minian\lib\site-packages\numpy\core\function_base.py in linspace(start, stop, num, endpoint, retstep, dtype, axis)
116 try:
--> 117 num = operator.index(num)
118 except TypeError:
TypeError: 'numpy.float64' object cannot be interpreted as an integer
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
in
~\Documents\GitHub\minian\minian\initialization.py in seeds_init(varr, wnd_size, method, stp_size, nchunk, max_wnd, diff_thres)
30 if method == 'rolling':
31 nstp = np.ceil(nfm / stp_size) + 1
---> 32 centers = np.linspace(0, nfm - 1, nstp)
33 hwnd = np.ceil(wnd_size / 2)
34 max_idx = list(
<array_function internals> in linspace(*args, **kwargs)
~\Anaconda3\envs\minian\lib\site-packages\numpy\core\function_base.py in linspace(start, stop, num, endpoint, retstep, dtype, axis)
117 num = operator.index(num)
118 except TypeError:
--> 119 raise TypeError(
120 "object of type {} cannot be safely interpreted as an integer."
121 .format(type(num)))
TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
my numpy version is 1.18
i found somewhere that downgrading it to 1.11 would fix the problem.
I used the conda env file fixed for correct xarray/dask versions
thank you!
richardson
Hi all,
I need to run minian on long recordings (at least 15 minutes sessions or ~27 video files) for my project. I have tried the pipeline many times using my university's HPC system and allocating up to 1.5TB of RAM to the process but I still face memory errors. And it happens early in the pipeline and in the "merge seeds" step.
when I break the data into ~5minute sessions, I can get it to work with the same RAM allocation. But I still need to get the pipeline working for the whole session.
Does anyone have a solution for that?
When running 4th cell, I get:
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
<ipython-input-4-8d76671ab02b> in <module>
1 sys.path.append(minian_path)
----> 2 from minian.utilities import load_params, load_videos, scale_varr, scale_varr_da, save_variable, open_minian, save_minian, handle_crash, get_optimal_chk
3 from minian.preprocessing import remove_brightspot, gradient_norm, denoise, remove_background, stripe_correction
4 from minian.motion_correction import estimate_shift_fft, apply_shifts, interpolate_frame, mask_shifts
5 from minian.initialization import seeds_init, gmm_refine, pnr_refine, intensity_refine, ks_refine, seeds_merge, initialize
~/minian/minian/utilities.py in <module>
18 import subprocess
19 import warnings
---> 20 import cv2
21 import papermill as pm
22 import ast
ImportError: libGL.so.1: cannot open shared object file: No such file or directory
It seems no cv2
.
implement some metric for the output quality of cells. Better done after #14 . Enabling better visualization as well. Ideas:
Event exceptionality. Ref: CaImAn paper CaImAn implementation
Spatial correlation at peaks. Ref: CaImAn paper
Isolation distance of cells with surrounding cells/pixels? Ref: Review paper, original definition of isolation distance
Requires #14 . Ideas:
It can be more user friendly (especially for people don't have much coding background) if there are highlight parts of note to tell user what they should be doing right now before starting run the next cell of code.
e.g:
1: a line of highlighted note saying: set dpath to the folder contains the recordings before the dpath setting part. Same thing, in batch processing, this line can be: ** set dpath to the master folder that contains all the recordings of animals/sessions you want to batch process here, number of folder layers doesn't matter.'**
2: a line of highlighted note saying: Applying the updated parameters to param_xxx now, for example, you can do: param_seeds_merger['thres_corr']=0.7 to apply the new parameter value. This line of note can locate after the parameter exploring part, and right before going into real updates.
Hi, I have a quick question: does minian have any kind of correction for photobleaching? For a long recording like 15-20mins.
I don't think I can do that with parameters, so curious if minian algorithm has it somewhere internally. I just wonder if it has or not.
Thanks!
Latest version of cvxpy (1.0.24) results in error during import minian
. Explicit download of v1.0.1 resolves issue. You may want to either explicitly put version control in .yml file or resolve compatibility issue with latest version.
testing pipeline on linear track data and find place cells. Can help showing:
Hey all!
The pipeline has been working great for single videos, however I am running into the following error when running it on a group of 20 (starting at smoothing signals, the entire terminal output of the pipeline is included here to show what steps are running successfully):
loading 20 videos in folder /work/miniscopepipeline/miniursi/videos/Animal_14/Day_4_FC
constructing chunks
creating parallel scheme
computing max projections
calculating local maximum
selecting seeds
computing peak-noise ratio
selecting seeds
performing KS test
computing distance
computing correlations
creating parallel schedule
computing ROIs
concatenating results
initializing backgrounds
estimating penalty parameter
computing subsetting matrix
fitting spatial matrix
removing empty units
updating background
background terms are empty
normalizing result
grouping overlaping units
computing trace
normalizing traces
smoothing signals
/home/eforbes/.local/lib/python3.6/site-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars
ret = ret.dtype.type(ret / rcount)
/work/miniscopepipeline/miniursi/minian/cnmf.py:60: RuntimeWarning: Mean of empty slice.
return np.sqrt(np.exp(np.log(px_band + eps).mean()))
estimating AR coefficients
Traceback (most recent call last):
File "ursi_pipeline.py", line 215, in
Y, A_spatial, b_spatial, C_spatial, f_spatial, sn_spatial, **param_first_temporal)
File "/work/miniscopepipeline/miniursi/minian/cnmf.py", line 497, in update_temporal
g = g.persist()
File "/home/eforbes/.local/lib/python3.6/site-packages/xarray/core/dataarray.py", line 845, in persist
ds = self._to_temp_dataset().persist(**kwargs)
File "/home/eforbes/.local/lib/python3.6/site-packages/xarray/core/dataset.py", line 848, in persist
return new._persist_inplace(**kwargs)
File "/home/eforbes/.local/lib/python3.6/site-packages/xarray/core/dataset.py", line 822, in _persist_inplace
evaluated_data = dask.persist(*lazy_data.values(), **kwargs)
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/base.py", line 630, in persist
results = schedule(dsk, keys, **kwargs)
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/threaded.py", line 84, in get
**kwargs
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/local.py", line 486, in get_async
raise_exception(exc, tb)
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/local.py", line 316, in reraise
raise exc
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/local.py", line 222, in execute_task
result = _execute_task(task, data)
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/core.py", line 119, in _execute_task
return func(*args2)
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/optimization.py", line 982, in call
return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/core.py", line 149, in get
result = _execute_task(task, cache)
File "/home/eforbes/.local/lib/python3.6/site-packages/dask/core.py", line 119, in _execute_task
return func(args2)
File "/home/eforbes/.local/lib/python3.6/site-packages/numpy/lib/function_base.py", line 2091, in call
return self._vectorize_call(func=func, args=vargs)
File "/home/eforbes/.local/lib/python3.6/site-packages/numpy/lib/function_base.py", line 2157, in _vectorize_call
res = self._vectorize_call_with_signature(func, args)
File "/home/eforbes/.local/lib/python3.6/site-packages/numpy/lib/function_base.py", line 2198, in _vectorize_call_with_signature
results = func((arg[index] for arg in args))
File "/work/miniscopepipeline/miniursi/minian/cnmf.py", line 648, in get_ar_coef
g = lstsq(C_mat, cov[1:max_lag + 1])[0]
File "/home/eforbes/.local/lib/python3.6/site-packages/scipy/linalg/basic.py", line 1157, in lstsq
a1 = _asarray_validated(a, check_finite=check_finite)
File "/home/eforbes/.local/lib/python3.6/site-packages/scipy/_lib/_util.py", line 246, in _asarray_validated
a = toarray(a)
File "/home/eforbes/.local/lib/python3.6/site-packages/numpy/lib/function_base.py", line 499, in asarray_chkfinite
"array must not contain infs or NaNs")
ValueError: array must not contain infs or NaNs
At first I had thought this error was due to there being too many seeds generated, but following the advice of an earlier issue I had adjusted those parameters. As far as I am aware, the environment is also set up properly as defined in the environment.yml file. Any help would be fantastic!
Thanks so much
Dear all,
This is more of a question rather than an issue that I wanted to ask you.
About the demo dataset, is it correct that we are using 10 recording files each having 1000 frames? And then we concatenate them all to form a complete recording session? If this is the case, shouldn't we expect to have 10 * 1000 = 10000 frames in our output processed files? Why is it that I only get 2000 frames in the output files?
Finally, since for my analysis, I am interested in spike trains from neurons, I want to use the S_temporal_it2 and then extract spike trains (e.g. a time series for each neuron/unit which states the times when a spike happens). So I want to get a single vector for each neuron which tells me when a spike happens. When I try to see what is stored in S_temporal_it2, I get:
And now I go back to my first question. Why is it that it shows only 2000 frames? I also see only 2000 frames in the plots of the last step in the pipeline. Am I missing something that I think we should get more frames from 10 recordings of 1000 frames? Similarly, Why do we only have a vector of length 1000 in S for each of 359 units extracted from demo videos? Shouldn't it be longer than that since we have concatenated 10 videos of each 1000 frames?
Also, could you please explain to me a bit about what is actually stored in this variable? What are the numbers in the array representative of? and can I get the spike trains (as I explained) from this pipeline and these saved variables? Should I be looking for all the local maxima in the rows of S each corresponding to a unit?
Thank you very much in advance for taking the time.
Is the netcdf file format still supported? In a previous version I was able to save as netcdf instead of zarr. There was a line in the parameters setting the backend to either zarr or netcdf. I tried adding this line back in, but was unable to get it to work. With netcdf I could pull data from the pipeline into matlab. Is there a way to still use the netcdf format or a better way to get the data into matlab after running through the pipeline? Thanks!!
After running motion correction on a 5000 frame sample set and saving the resulting shifts variable, I get a codec error that I haven't gotten on smaller data sets.
Line: save_minian(shifts.rename('shifts'), **param_save_minian)
ValueError Traceback (most recent call last)
in
~\minian\minian\utilities.py in save_minian(var, dpath, fname, backend, meta_dict, overwrite)
694 elif backend is 'zarr':
695 md = {True: 'w', False: 'w-'}[overwrite]
--> 696 ds.to_zarr(os.path.join(dpath, fname, var.name + '.zarr'), mode=md)
697 return ds
698 else:
...................................
~\Anaconda3\envs\minian\lib\site-packages\numcodecs\compat.py in ensure_contiguous_ndarray(buf, max_buffer_size)
140 if max_buffer_size is not None and arr.nbytes > max_buffer_size:
141 msg = "Codec does not support buffers of > {} bytes".format(max_buffer_size)
--> 142 raise ValueError(msg)
143
144 return arr
ValueError: Codec does not support buffers of > 2147483647 bytes
It might be more clear and easy to understand if the final video which will be write into local folder can have some title/label for each of those 4 things. E.g, it's helpful to know the 'raw video' here is supposed already been motion corrected, if your 'raw' is still shifting, then that will indicate you need to go back and fix something.
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.