Reconstructing a complete scan using nabu and config file, from python¶

In previous notebooks nabu_basic_reconstruction.ipynb and nabu_from_python_with_gpu.ipynb, we saw how to manipulate the individual processing classes for step-by-step processing (eg. FlatField, PaganinPhaseRetrieval ; and their cuda counterparts).

In this notebook, we see how to use a facility class FullFieldReconstructor which assembles all these steps, does the "boring work" for you (memory allocation, data transfers, configuration ingestion, ...). It reconstructs the input volume by parts, handling the necessary overlap between sub-volumes if needed.

1. Get the scan (.nx file) and processing configuration (.conf file)¶

In [1]:

import os
from nabu.testutils import utilstest, get_file
from nabu.pipeline.fullfield.processconfig import ProcessConfig

import os from nabu.testutils import utilstest, get_file from nabu.pipeline.fullfield.processconfig import ProcessConfig

In [2]:

print("Getting dataset (downloading if necessary) ...")
data_path = get_file("bamboo_reduced.nx")
print("... OK")

# Get the configuration file of this dataset
conf_fname = get_file("bamboo_reduced.conf")

# Change directory to the path where the data is located (only useful for this tutorial)
os.chdir(utilstest.data_home)

# Parse this configuration file
conf = ProcessConfig(conf_fname)

print("Getting dataset (downloading if necessary) ...") data_path = get_file("bamboo_reduced.nx") print("... OK") # Get the configuration file of this dataset conf_fname = get_file("bamboo_reduced.conf") # Change directory to the path where the data is located (only useful for this tutorial) os.chdir(utilstest.data_home) # Parse this configuration file conf = ProcessConfig(conf_fname)

Getting dataset (downloading if necessary) ...

ERROR:nabu:Cannot do SRCurrent normalization: missing flats and/or projections SRCurrent

... OK
Option 'double_flatfield_enabled' has been renamed 'double_flatfield' in [preproc]
This is deprecated since version 2025.1.0 and will result in an error in futures versions

In [ ]:

2. Running volume reconstruction¶

You can specify which reconstruction backend to use ("numpy" or "cuda")

In [3]:

from nabu.pipeline.fullfield.reconstruction import FullFieldReconstructor 

from nabu.pipeline.fullfield.reconstruction import FullFieldReconstructor

In [4]:

reconstructor = FullFieldReconstructor(conf, backend="cuda")

reconstructor = FullFieldReconstructor(conf, backend="cuda")

In [5]:

reconstructor.reconstruct()

reconstructor.reconstruct()

[0] Processing sub-volume ((0, 1000), (0, 540), (0, 640))

[0] Something went wrong with task 0 ! Aborting reconstruction

Traceback (most recent call last):
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/tasks.py", line 185, in start_work
    result = self._processor.process_task(task)
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/reconstruction.py", line 148, in process_task
    self._instantiate_pipeline_if_necessary(task, prev_task)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/reconstruction.py", line 120, in _instantiate_pipeline_if_necessary
    self._instantiate_pipeline(current_task)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/reconstruction.py", line 112, in _instantiate_pipeline
    pipeline = self._pipeline_cls(*args, margin=task["margin"], **pipeline_init_kwargs)
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/chunked_cuda.py", line 69, in __init__
    super().__init__(
    ~~~~~~~~~~~~~~~~^
        process_config,
        ^^^^^^^^^^^^^^^
    ...<3 lines>...
        margin=margin,
        ^^^^^^^^^^^^^^
    )
    ^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/chunked.py", line 112, in __init__
    self._init_pipeline()
    ~~~~~~~~~~~~~~~~~~~^^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/chunked.py", line 291, in _init_pipeline
    self._init_flatfield()
    ~~~~~~~~~~~~~~~~~~~~^^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/utils.py", line 24, in wrapper
    return func(*args, **kwargs)
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/chunked.py", line 472, in _init_flatfield
    self.flatfield = self.FlatFieldClass(
                     ~~~~~~~~~~~~~~~~~~~^
        self.radios_shape,
        ^^^^^^^^^^^^^^^^^^
    ...<6 lines>...
        flats_srcurrent=self._ff_options["flats_srcurrent"],
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/preproc/flatfield_cuda.py", line 44, in __init__
    self._init_cuda_kernels()
    ~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/preproc/flatfield_cuda.py", line 59, in _init_cuda_kernels
    self.cuda_kernel = self.cuda_processing.kernel(
                       ~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        "flatfield_normalization", self._cuda_fname, options=tuple(options)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/cuda/processing.py", line 66, in kernel
    return CudaKernel(  # pylint: disable=E0606
        kernel_name,
    ...<3 lines>...
        **build_kwargs,
    )
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/cuda/kernel.py", line 48, in __init__
    self.compile_kernel_source(kernel_name, sourcemodule_kwargs)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/nabu/cuda/kernel.py", line 59, in compile_kernel_source
    self.module.compile()
    ~~~~~~~~~~~~~~~~~~~^^
  File "cupy/_core/raw.pyx", line 437, in cupy._core.raw.RawModule.compile
  File "cupy/_core/raw.pyx", line 415, in cupy._core.raw.RawModule._module
  File "cupy/_util.pyx", line 68, in cupy._util.memoize.decorator.ret
  File "cupy/_core/raw.pyx", line 549, in cupy._core.raw._get_raw_module
  File "cupy/_core/core.pyx", line 2534, in cupy._core.core.compile_with_cache
  File "cupy/_core/core.pyx", line 2552, in cupy._core.core.compile_with_cache
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/cupy/cuda/compiler.py", line 566, in _compile_module_with_cache
    return _compile_with_cache_cuda(
        source, options, arch, cache_dir, extra_source, backend,
        enable_cooperative_groups, name_expressions, log_stream,
        cache_in_memory, jitify, to_ltoir)
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/cupy/cuda/compiler.py", line 618, in _compile_with_cache_cuda
    base = _preprocess('', options, arch, backend)
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/cupy/cuda/compiler.py", line 513, in _preprocess
    result = compile_using_nvcc(source, options, arch, 'preprocess.cu',
                                code_type='ptx')
  File "/home/pierre/.venv/py313/lib/python3.13/site-packages/cupy/cuda/compiler.py", line 425, in compile_using_nvcc
    cmd = _nvcc.split()
          ^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'split'

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
File ~/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/reconstruction.py:747, in FullFieldReconstructor._reconstruct_chunked(self, tasks)
    746             logger.fatal(msg)
--> 747             raise RuntimeError(msg)
    749 finally:
    750     # Terminate workers

RuntimeError: Something went wrong with task 0 ! Aborting reconstruction

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
Cell In[5], line 1
----> 1 reconstructor.reconstruct()

File ~/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/reconstruction.py:787, in FullFieldReconstructor.reconstruct(self)
    785 def reconstruct(self):
    786     if self._pipeline_mode == "chunked":
--> 787         self._reconstruct_chunked()
    788     else:
    789         self._reconstruct_grouped()

File ~/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/reconstruction.py:756, in FullFieldReconstructor._reconstruct_chunked(self, tasks)
    754 resident_pipeline = getattr(self.tasks_processor, "_processor", None)
    755 if resident_pipeline is not None and self.extra_options.get("cleanup_memory", True):
--> 756     resident_pipeline._destroy_pipeline()

File ~/.venv/py313/lib/python3.13/site-packages/nabu/pipeline/fullfield/reconstruction.py:130, in NannyPipeline._destroy_pipeline(self)
    129 def _destroy_pipeline(self):
--> 130     pipeline_backend = self.pipeline.backend
    131     self.pipeline = None
    132     if pipeline_backend != "cuda":

AttributeError: 'NoneType' object has no attribute 'backend'

In [6]:

output_file = reconstructor.merge_hdf5_reconstructions()

output_file = reconstructor.merge_hdf5_reconstructions()

[0] No files to merge

3. Visualization¶

In [7]:

from tomoscan.io import HDF5File

from tomoscan.io import HDF5File

In [8]:

with HDF5File(output_file, "r") as f:
    data_ptr = f["entry0000/reconstruction/results/data"]
    middle_slice_horiz = data_ptr[reconstructor.delta_z//2]
    middle_slice_vertic = data_ptr[:, :, reconstructor.n_x//2]
    

with HDF5File(output_file, "r") as f: data_ptr = f["entry0000/reconstruction/results/data"] middle_slice_horiz = data_ptr[reconstructor.delta_z//2] middle_slice_vertic = data_ptr[:, :, reconstructor.n_x//2]

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[8], line 1
----> 1 with HDF5File(output_file, "r") as f:
      2     data_ptr = f["entry0000/reconstruction/results/data"]
      3     middle_slice_horiz = data_ptr[reconstructor.delta_z//2]

File ~/.venv/py313/lib/python3.13/site-packages/h5py/_hl/files.py:530, in File.__init__(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, alignment_threshold, alignment_interval, meta_block_size, track_times, **kwds)
    528     name = repr(name).encode('ASCII', 'replace')
    529 else:
--> 530     name = filename_encode(name)
    532 if track_order is None:
    533     track_order = h5.get_config().track_order

File ~/.venv/py313/lib/python3.13/site-packages/h5py/_hl/compat.py:25, in filename_encode(filename)
     17 def filename_encode(filename):
     18     """
     19     Encode filename for use in the HDF5 library.
     20 
   (...)     23     filenames in h5py for more information.
     24     """
---> 25     filename = os.fspath(filename)
     26     if sys.platform == "win32" and isinstance(filename, str):
     27         return filename.encode(WINDOWS_ENCODING, "strict")

TypeError: expected str, bytes or os.PathLike object, not NoneType

In [9]:

import matplotlib.pyplot as plt
plt.figure()
plt.subplot(121)
plt.imshow(middle_slice_horiz, cmap="gray")
plt.xlabel("Horizontal slice, middle")
plt.subplot(122)
plt.imshow(middle_slice_vertic, cmap="gray")
plt.xlabel("Vertical slice, middle")
plt.show()

import matplotlib.pyplot as plt plt.figure() plt.subplot(121) plt.imshow(middle_slice_horiz, cmap="gray") plt.xlabel("Horizontal slice, middle") plt.subplot(122) plt.imshow(middle_slice_vertic, cmap="gray") plt.xlabel("Vertical slice, middle") plt.show()

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[9], line 4
      2 plt.figure()
      3 plt.subplot(121)
----> 4 plt.imshow(middle_slice_horiz, cmap="gray")
      5 plt.xlabel("Horizontal slice, middle")
      6 plt.subplot(122)

NameError: name 'middle_slice_horiz' is not defined

In [ ]: