From 9f9178cec7d0c6c2acd933b7a66770162e925a25 Mon Sep 17 00:00:00 2001
From: Beliy Nikita <beliy.nikita@outlook.com>
Date: Tue, 18 Aug 2020 13:17:59 +0200
Subject: [PATCH] Updated example to current version
---
example1/README.md | 23 +++++--
example1/resources/map/bidsmap.yaml | 35 ++++++----
example1/resources/plugins/bidsify_plugin.py | 53 +++++++--------
example1/resources/plugins/process_plugin.py | 71 ++++++++++----------
example1/resources/plugins/rename_plugin.py | 25 +++++--
5 files changed, 121 insertions(+), 86 deletions(-)
diff --git a/example1/README.md b/example1/README.md
index d16fe0d..b8155d7 100644
--- a/example1/README.md
+++ b/example1/README.md
@@ -289,10 +289,12 @@ In this step, a generic user-defined dataset is organized in a standardized way.
To run data preparation, it will be enough to run from `example1` directory
```python
-python3 bidsme.py prepare --part-template resources/participants.json --recfolder nii=MRI --plugin resources/plugins/rename_plugin.py source/ renamed/
+python3 bidsme.py prepare --part-template resources/participants.json --recfolder nii=MRI --plugin resources/plugins/rename_plugin.py -- source/ renamed/
```
-The options `--part-template resources/participants.json` will tell bidsme to use participant json file as template for `participants.tsv` file.
+The options `--part-template resources/participants.json` will tell bidsme to use participant json file as template for `participants.tsv` file.
+The template json file must follow the bids specification for tsv tables definitions, found
+[there](https://bids-specification.readthedocs.io/en/stable/02-common-principles.html#tabular-files).
The column `participant_id` will be filled automatically, while other columns will be filled
by default by `n/a`, unless they are set in plugin:
@@ -302,13 +304,26 @@ session.sub_values["sex"] = "M"
Without `--part-template` option the only column in participants file will be `participant_id`.
-Option `--recfolder nii=MRI` will tell to `bidsme` that image files are MRI and stored in `nii` folder.
+Option `-r nii=MRI` will tell to `bidsme` that image files are MRI and stored in `nii` folder.
Without this option `bidsme` will be unable to find image files.
+This option can accept a list of arguments if data-files are stored in different sub-folders,
+for example:
+```
+-r <path1>=<data_type_1> <path2>=<data_type_2>
+```
+In this case `bidsme` will search data file of given type in `<subject>/<session>/<path1>` folder.
+Paths can accept basic shell expensions as defined in [glob module](https://docs.python.org/3.6/library/glob.html).
+
Option `--plugin resources/plugins/rename-plugin.py` will tell to bidsme to load corresponding plugin.
Parameters `source/` and `renamed/` tells to bidsme where to search for source dataset and where place prepared dataset.
+> Several options accepts a list of arguments. If such options are followed by positional
+arguments (`source/` and `renamed/` in this case), it will confuse the programm.
+To avoid this, positional arguments must be preseeded by double dash (`--`), that
+marks end of optional arguments, and beginning of positional ones
+
After the execution of preparation, the `rename` folder should contain folders and files:
- **code/bidsme**, with log files of the last execution of preparation step
@@ -371,7 +386,7 @@ do not exists in bids dataset.
So it can be used as check before bidsification.
With plugins, it can be used for data manipulation, and metadata completion.
-For example `resources/plugins/process_plugin.py` fills the `nandiness` column, and merges
+For example `resources/plugins/process_plugin.py` fills the `handiness` column, and merges
fMRI and diffusion images in single 4D image.
```python
diff --git a/example1/resources/map/bidsmap.yaml b/example1/resources/map/bidsmap.yaml
index d659c6a..0db0db3 100644
--- a/example1/resources/map/bidsmap.yaml
+++ b/example1/resources/map/bidsmap.yaml
@@ -13,7 +13,7 @@ MRI:
checked: true
suffix: ''
attributes:
- SeriesDescription: 'invalid'
+ <<custom:IntendedFor>>: 'invalid'
bids: !!omap []
json: !!omap []
func:
@@ -24,7 +24,7 @@ MRI:
attributes:
ProtocolName: cmrr_mbep2d_bold_mb2_invertpe
ImageType: ORIGINAL\\PRIMARY\\M\\MB\\ND\\MOSAIC
- SeriesDescription: 'nBack'
+ <<custom:IntendedFor>>: 'nBack'
bids: !!omap
- dir: PA
- task: nBack
@@ -46,7 +46,7 @@ MRI:
attributes:
ProtocolName: cmrr_mbep2d_bold_mb2_invertpe
ImageType: ORIGINAL\\PRIMARY\\M\\MB\\ND\\MOSAIC
- SeriesDescription: 'rest'
+ <<custom:IntendedFor>>: 'rest'
bids: !!omap
- dir: PA
- task: rest
@@ -132,7 +132,7 @@ MRI:
attributes:
ProtocolName: 'gre_field_mapping'
ImageType: 'ORIGINAL\\PRIMARY\\M\\ND'
- SeriesDescription: 'HCL/LCL'
+ <<custom:IntendedFor>>: 'HCL/LCL'
bids: !!omap
- acq: ~
- ce: ~
@@ -150,7 +150,7 @@ MRI:
attributes:
ProtocolName: 'gre_field_mapping'
ImageType: 'ORIGINAL\\PRIMARY\\M\\ND'
- SeriesDescription: 'STROOP'
+ <<custom:IntendedFor>>: 'STROOP'
bids: !!omap
- acq: ~
- ce: ~
@@ -169,7 +169,7 @@ MRI:
attributes:
ProtocolName: 'gre_field_mapping'
ImageType: 'ORIGINAL\\PRIMARY\\P\\ND'
- SeriesDescription: 'HCL/LCL'
+ <<custom:IntendedFor>>: 'HCL/LCL'
bids: !!omap
- acq: ~
- ce: ~
@@ -189,7 +189,7 @@ MRI:
attributes:
ProtocolName: 'gre_field_mapping'
ImageType: 'ORIGINAL\\PRIMARY\\P\\ND'
- SeriesDescription: 'STROOP'
+ <<custom:IntendedFor>>: 'STROOP'
bids: !!omap
- acq: ~
- ce: ~
@@ -209,7 +209,7 @@ MRI:
suffix: B1minus
attributes:
ProtocolName: 'al_mtflash3d_sensArray'
- SeriesDescription: 'PDw'
+ <<custom:IntendedFor>>: 'PDw'
bids: !!omap
- acq: HeadPDw
- mod: ~
@@ -223,7 +223,7 @@ MRI:
suffix: B1minus
attributes:
ProtocolName: 'al_mtflash3d_sensBody'
- SeriesDescription: 'PDw'
+ <<custom:IntendedFor>>: 'PDw'
bids: !!omap
- acq: BodyPDw
- mod: ~
@@ -243,25 +243,34 @@ MRI:
json: !!omap
- EchoTime: <scale-3:EchoTime>
- B1mapNominalFAValues: <B1mapNominalFAValues>
+ - IntendedFor:
+ - "fmap/<<subject>>_<<session>>_acq-HeadPDw_B1minus.nii"
+ - "fmap/<<subject>>_<<session>>_acq-BodyPDw_B1minus.nii"
+ - "fmap/<<subject>>_<<session>>_acq-HeadT1w_B1minus.nii"
+ - "fmap/<<subject>>_<<session>>_acq-BodyT1w_B1minus.nii"
+ - "fmap/<<subject>>_<<session>>_acq-HeadMTw_B1minus.nii"
+ - "fmap/<<subject>>_<<session>>_acq-BodyMTw_B1minus.nii"
- provenance: /home/beliy/Works/bidscoin/test/renamed/sub-003/ses-STROOP/MRI/006-al_mtflash3d_sensArray/s1905-0006-00001-000032-01.nii
example: fmap/sub-003_ses-STROOP_acq-HeadT1w_B1minus
checked: true
suffix: B1minus
attributes:
ProtocolName: 'al_mtflash3d_sensArray'
- SeriesDescription: 'T1w'
+ <<custom:IntendedFor>>: 'T1w'
bids: !!omap
- acq: HeadT1w
- mod: ~
json: !!omap
- EchoTime: <scale-3:EchoTime>
+ - IntendedFor:
+ - "anat/<<subject>>_<<session>>_acq-<<bids:mod>>_echo-*_MPM.nii"
- provenance: /home/beliy/Works/bidscoin/test/renamed/sub-003/ses-STROOP/MRI/007-al_mtflash3d_sensBody/s1905-0007-00001-000032-01.nii
example: fmap/sub-003_ses-STROOP_acq-BodyT1w_B1minus
checked: true
suffix: B1minus
attributes:
ProtocolName: 'al_mtflash3d_sensBody'
- SeriesDescription: 'T1w'
+ <<custom:IntendedFor>>: 'T1w'
bids: !!omap
- acq: BodyT1w
- mod: ~
@@ -275,7 +284,7 @@ MRI:
suffix: B1minus
attributes:
ProtocolName: 'al_mtflash3d_sensArray'
- SeriesDescription: 'MTw'
+ <<custom:IntendedFor>>: 'MTw'
bids: !!omap
- acq: HeadMTw
- mod: ~
@@ -289,7 +298,7 @@ MRI:
suffix: B1minus
attributes:
ProtocolName: 'al_mtflash3d_sensBody'
- SeriesDescription: 'MTw'
+ <<custom:IntendedFor>>: 'MTw'
bids: !!omap
- acq: BodyMTw
- mod: ~
diff --git a/example1/resources/plugins/bidsify_plugin.py b/example1/resources/plugins/bidsify_plugin.py
index 06255b2..3464cbe 100644
--- a/example1/resources/plugins/bidsify_plugin.py
+++ b/example1/resources/plugins/bidsify_plugin.py
@@ -5,6 +5,10 @@ import random
from definitions import checkSeries
+"""
+bidsify_plugin defines all nessesary functions to bidsify
+prepared dataset
+"""
# defining logger this way will prefix plugin messages
# with plugin name
@@ -42,10 +46,6 @@ seq_list = list()
# The index of current sequence, corresponds to order in the sequence list
seq_index = -1
-# Identified tag for fMRI and MPM MRI
-# This tag will override "SeriesDescription" DICOM tag
-IntendedFor = ""
-
def InitEP(source: str, destination: str, dry: bool) -> int:
"""
@@ -180,8 +180,12 @@ def SequenceEP(recording):
Sequence identification
"""
global seq_index
- global IntendedFor
- IntendedFor = ""
+
+ # recording.custom is a dictionary for user-defined variables
+ # that can be acessed from bidsmap
+ # they are initialized at new sequence, and conserved for all files
+ # within sequence, can be used to define sequence-global parameters
+ recording.custom["IntendedFor"] = ""
seq_index += 1
recid = seq_list[seq_index]
@@ -196,13 +200,13 @@ def SequenceEP(recording):
if recid == "cmrr_mbep2d_bold_mb2_invertpe":
mod = seq_list[seq_index + 1]
if mod.endswith("cmrr_mbep2d_bold_mb2_task_fat"):
- IntendedFor = "nBack"
+ recording.custom["IntendedFor"] = "nBack"
elif mod.endswith("cmrr_mbep2d_bold_mb2_task_nfat"):
- IntendedFor = "nBack"
+ recording.custom["IntendedFor"] = "nBack"
elif mod.endswith("cmrr_mbep2d_bold_mb2_rest"):
- IntendedFor = "rest"
+ recording.custom["IntendedFor"] = "rest"
else:
- IntendedFor = "invalid"
+ recording.custom["IntendedFor"] = "invalid"
logger.warning("{}: Unknown session {}"
.format(recording.recIdentity(),
mod))
@@ -210,43 +214,38 @@ def SequenceEP(recording):
# sessions
elif recid == "gre_field_mapping":
if recording.sesId() in ("ses-HCL", "ses-LCL"):
- IntendedFor = "HCL/LCL"
+ recording.custom["IntendedFor"] = "HCL/LCL"
elif recording.sesId() == "ses-STROOP":
- IntendedFor = "STROOP"
+ recording.custom["IntendedFor"] = "STROOP"
else:
logger.warning("{}: Unknown session {}"
.format(recording.recIdentity(),
recording.sesId()))
- IntendedFor = "invalid"
- # fmaps sesnsBody and sesnArray are taken just before
+ recording.custom["IntendedFor"] = "invalid"
+ # fmaps sensBody and sensArray are taken just before
# structural PD , T1 and MT. Looking into next sequences
# will allow the identification
elif recid == "al_mtflash3d_sensArray":
det = seq_list[seq_index + 2]
if det.endswith("al_mtflash3d_PDw"):
- IntendedFor = "PDw"
+ recording.custom["IntendedFor"] = "PDw"
elif det.endswith("al_mtflash3d_T1w"):
- IntendedFor = "T1w"
+ recording.custom["IntendedFor"] = "T1w"
elif det.endswith("al_mtflash3d_MTw"):
- IntendedFor = "MTw"
+ recording.custom["IntendedFor"] = "MTw"
else:
logger.warning("{}: Unable determine modality"
.format(recording.recIdentity()))
- IntendedFor = "invalid"
+ recording.custom["IntendedFor"] = "invalid"
elif recid == "al_mtflash3d_sensBody":
det = seq_list[seq_index + 1]
if det.endswith("al_mtflash3d_PDw"):
- IntendedFor = "PDw"
+ recording.custom["IntendedFor"] = "PDw"
elif det.endswith("al_mtflash3d_T1w"):
- IntendedFor = "T1w"
+ recording.custom["IntendedFor"] = "T1w"
elif det.endswith("al_mtflash3d_MTw"):
- IntendedFor = "MTw"
+ recording.custom["IntendedFor"] = "MTw"
else:
logger.warning("{}: Unable determine modality"
.format(recording.recIdentity()))
- IntendedFor = "invalid"
-
-
-def RecordingEP(recording):
- if IntendedFor != "":
- recording.setAttribute("SeriesDescription", IntendedFor)
+ recording.custom["IntendedFor"] = "invalid"
diff --git a/example1/resources/plugins/process_plugin.py b/example1/resources/plugins/process_plugin.py
index f8a42f5..758bb18 100644
--- a/example1/resources/plugins/process_plugin.py
+++ b/example1/resources/plugins/process_plugin.py
@@ -3,8 +3,14 @@ import shutil
import logging
import random
+from bids import BidsSession
+
from definitions import checkSeries, plugin_root
+"""
+process_plugin defines all nessesary functions to pre-process
+prepared dataset. Essentually it just merges 3D images to 4D
+"""
# defining logger this way will prefix plugin messages
# with plugin name
@@ -27,8 +33,7 @@ dry_run = False
#####################
# Some sequences within session (namely fMRI and MPM structural) follows same
-# protocol, thus it is impossible to identify them only using
-# metadata
+# protocol, thus it is impossible to identify them only using metadata
# we will identify them by order they appear in session
# list of sequences in order of acquisition in current session
@@ -42,10 +47,6 @@ seq_list = list()
# The index of current sequence, corresponds to order in the sequence list
seq_index = -1
-# Identified tag for fMRI and MPM MRI
-# This tag will override "SeriesDescription" DICOM tag
-IntendedFor = ""
-
def InitEP(source: str, destination: str, dry: bool) -> int:
"""
@@ -69,7 +70,7 @@ def InitEP(source: str, destination: str, dry: bool) -> int:
dry_run = dry
-def SubjectEP(scan):
+def SubjectEP(scan: BidsSession) -> int:
"""
Subject modification
@@ -79,11 +80,11 @@ def SubjectEP(scan):
# values in scan.sub_values are pre-filled
# from participants.tsv file, no need to refill them.
# If needed to remove value from participants, you
- # can set it to None
+ # can set it to None, or set to new value
scan.sub_values["handiness"] = random.choice([0, 1])
-def SessionEP(scan):
+def SessionEP(scan: BidsSession) -> int:
"""
Session files modifications
@@ -96,11 +97,12 @@ def SessionEP(scan):
######################################
# Initialisation of sesion variables #
######################################
- # retrieving list of sequences and puttintg them into list
+ # retrieving sequences and puttintg them into list
global seq_list
global seq_index
path = os.path.join(scan.in_path, "MRI")
seq_list = sorted(os.listdir(path))
+ # removing leading sequence number from name
seq_list = [s.split("-", 1)[1] for s in seq_list]
seq_index = -1
@@ -112,6 +114,7 @@ def SessionEP(scan):
#############################################
# Checking for existance of auxiliary files #
#############################################
+ # BidsSession.in_path contains current session folder path
aux_input = os.path.join(scan.in_path, "auxiliary")
if scan.session in ("ses-LCL", "ses-HCL"):
if not os.path.isdir(aux_input):
@@ -134,8 +137,12 @@ def SequenceEP(recording):
"""
global seq_index
- global IntendedFor
- IntendedFor = ""
+
+ # recording.custom is a dictionary for user-defined variables
+ # that can be acessed from bidsmap
+ # they are initialized at new sequence, and conserved for all files
+ # within sequence, can be used to define sequence-global parameters
+ recording.custom["IntendedFor"] = ""
seq_index += 1
recid = seq_list[seq_index]
@@ -151,13 +158,13 @@ def SequenceEP(recording):
if recid == "cmrr_mbep2d_bold_mb2_invertpe":
mod = seq_list[seq_index + 1]
if mod.endswith("cmrr_mbep2d_bold_mb2_task_fat"):
- IntendedFor = "nBack"
+ recording.custom["IntendedFor"] = "nBack"
elif mod.endswith("cmrr_mbep2d_bold_mb2_task_nfat"):
- IntendedFor = "nBack"
+ recording.custom["IntendedFor"] = "nBack"
elif mod.endswith("cmrr_mbep2d_bold_mb2_rest"):
- IntendedFor = "rest"
+ recording.custom["IntendedFor"] = "rest"
else:
- IntendedFor = "invalid"
+ recording.custom["IntendedFor"] = "invalid"
logger.warning("{}: Unknown session {}"
.format(recording.recIdentity(),
mod))
@@ -165,49 +172,41 @@ def SequenceEP(recording):
# sessions
elif recid == "gre_field_mapping":
if recording.sesId() in ("ses-HCL", "ses-LCL"):
- IntendedFor = "HCL/LCL"
+ recording.custom["IntendedFor"] = "HCL/LCL"
elif recording.sesId() == "ses-STROOP":
- IntendedFor = "STROOP"
+ recording.custom["IntendedFor"] = "STROOP"
else:
logger.warning("{}: Unknown session {}"
.format(recording.recIdentity(),
recording.sesId()))
- IntendedFor = "invalid"
+ recording.custom["IntendedFor"] = "invalid"
# fmaps sesnsBody and sesnArray are taken just before
# structural PD , T1 and MT. Looking into next sequences
# will allow the identification
elif recid == "al_mtflash3d_sensArray":
det = seq_list[seq_index + 2]
if det.endswith("al_mtflash3d_PDw"):
- IntendedFor = "PDw"
+ recording.custom["IntendedFor"] = "PDw"
elif det.endswith("al_mtflash3d_T1w"):
- IntendedFor = "T1w"
+ recording.custom["IntendedFor"] = "T1w"
elif det.endswith("al_mtflash3d_MTw"):
- IntendedFor = "MTw"
+ recording.custom["IntendedFor"] = "MTw"
else:
logger.warning("{}: Unable determine modality"
.format(recording.recIdentity()))
- IntendedFor = "invalid"
+ recording.custom["IntendedFor"] = "invalid"
elif recid == "al_mtflash3d_sensBody":
det = seq_list[seq_index + 1]
if det.endswith("al_mtflash3d_PDw"):
- IntendedFor = "PDw"
+ recording.custom["IntendedFor"] = "PDw"
elif det.endswith("al_mtflash3d_T1w"):
- IntendedFor = "T1w"
+ recording.custom["IntendedFor"] = "T1w"
elif det.endswith("al_mtflash3d_MTw"):
- IntendedFor = "MTw"
+ recording.custom["IntendedFor"] = "MTw"
else:
logger.warning("{}: Unable determine modality"
.format(recording.recIdentity()))
- IntendedFor = "invalid"
-
-
-def RecordingEP(recording):
- """
- Setting "SeriesDescription" tag for given recording.
- """
- if IntendedFor != "":
- recording.setAttribute("SeriesDescription", IntendedFor)
+ recording.custom["IntendedFor"] = "invalid"
def SequenceEndEP(outfolder, recording):
@@ -227,7 +226,7 @@ def SequenceEndEP(outfolder, recording):
modality))
first_file = os.path.join(outfolder, recording.files[0])
# "convertion" is just copy of first file in sequence
- # in real application a real external tool should be used
+ # in real application a external tool should be used
shutil.copy2(first_file, f4D + ".nii")
first_file = os.path.splitext(first_file)[0] + ".json"
# copying the first file json to allow the identification
diff --git a/example1/resources/plugins/rename_plugin.py b/example1/resources/plugins/rename_plugin.py
index e0be9bf..02589b5 100644
--- a/example1/resources/plugins/rename_plugin.py
+++ b/example1/resources/plugins/rename_plugin.py
@@ -8,6 +8,11 @@ from bids import BidsSession
from definitions import Series, checkSeries, plugin_root
+"""
+rename_plugin defines all nessesary functions to prepare source
+dataset. In particular it identifies the correct session id and
+retrieves demographic, task and assesment data
+"""
# defining logger this way will prefix plugin messages
# with plugin name
@@ -43,7 +48,7 @@ time_scale = 1e-3
# by plugin
sub_black_list = []
-# subject xls table columns and their renaiming
+# subject xls table columns and their renaming
excel_col_list = {"Patient": "pat",
"Sex": "pat_sex",
"Age": "pat_age",
@@ -157,7 +162,7 @@ def SubjectEP(session: BidsSession) -> int:
return -1
# storing bidsified subject id into session object
- # optional, but can be easely retrieved
+ # optional, but useful as reference
session.sub_values["participant_id"] = "sub-" + session.subject
# looking for subject in dataframe
prefix = "pat"
@@ -197,7 +202,9 @@ def SubjectEP(session: BidsSession) -> int:
if pandas.notna(paired):
session.sub_values["paired"] = "sub-{:03}".format(int(paired))
- # looking for order of sessions
+ #################################
+ # determining order of sessions #
+ #################################
scans_map.clear()
scans_order = sorted([os.path.basename(s) for s in
tools.lsdirs(os.path.join(rawfolder,
@@ -221,8 +228,8 @@ def SubjectEP(session: BidsSession) -> int:
# participant left study
logger.warning("Subject {}({}): seems to be abandoned study"
.format(session.sub_values["participant_id"],
- session.sub_values["group"],
- ses)
+ session.sub_values["group"]
+ )
)
return -1
elif v not in Series:
@@ -264,7 +271,7 @@ def SessionEP(session: BidsSession) -> int:
----------
session: BidsSession
"""
- # Setting session name from map
+ # Renaming session name from map
session.session = scans_map[session.session]
@@ -276,6 +283,12 @@ def SessionEndEP(session: BidsSession):
"""
# path contain destination folder, where
# all data files are placed
+
+ # session.getPath generates bids path based on
+ # subject and session id, eg. sub-001/ses-HCL
+ # if parameter empty==True, and no session,
+ # the generated path will still contain 'ses-'
+ # empty must be True in preparation plugin
path = os.path.join(preparefolder,
session.getPath(True))
out_path = os.path.join(path,
--
GitLab