Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
138 commits
Select commit Hold shift + click to select a range
d3d7f0d
Exporting OTF data to CryoSparc
delarosatrevin Dec 3, 2024
e8d7670
Some updates in ProcessingPipeline
delarosatrevin Dec 19, 2024
21b699f
Added context manager for tmpDir in Path class
delarosatrevin Jan 4, 2025
4cba58b
Created Batch class as a utility subclass from dict
delarosatrevin Jan 4, 2025
c483317
Added helper Args class
delarosatrevin Jan 5, 2025
f2d5413
Bumped devel version
delarosatrevin Jan 5, 2025
d31dfa8
Allow to pass input to process as stdin
delarosatrevin Jan 5, 2025
99a6ae1
Added functions to Batch class
delarosatrevin Jan 7, 2025
17593cf
Some fixs in Args
delarosatrevin Jan 7, 2025
b4543fa
Minor fixes and new Table.fromDict function
delarosatrevin Jan 9, 2025
2eae5bf
Moved ProcessingPipeline from emtools to emwrap
delarosatrevin Jan 9, 2025
9a6cb03
Introduced FolderManager class
delarosatrevin Jan 11, 2025
918d65f
Check if folder exists
delarosatrevin Jan 14, 2025
6844864
Added helper function StarFile.getTableFromFile
delarosatrevin Jan 14, 2025
bb3fad0
added emt-star utility program
delarosatrevin Jan 15, 2025
125c3e2
Allow to dump json files in the folder
delarosatrevin Jan 24, 2025
d9fba20
Add more methods to Batch
delarosatrevin Jan 27, 2025
a34fc35
Used another threading.Condition to allow maxsize in TaskQueue
delarosatrevin Jan 31, 2025
7277664
Added utility to read image dimensions
delarosatrevin Feb 3, 2025
bd80dbd
Added 'group_by' option to emt-star
delarosatrevin Feb 3, 2025
19a12b6
Added prototype to split star files
delarosatrevin Feb 5, 2025
c9e710b
Allow to pass print function
delarosatrevin Feb 6, 2025
6d23991
Added more logging to Batch
delarosatrevin Feb 6, 2025
0f88eea
Allow missing coordinate files
delarosatrevin Feb 6, 2025
39f1f6f
Added queueMaxSize for TaskProcessor
delarosatrevin Feb 6, 2025
8ec00bb
Moved create to FolderManager
delarosatrevin Feb 7, 2025
3193b16
Added methods to Mdoc
delarosatrevin Feb 7, 2025
69ea1ca
Moved create to FolderManager
delarosatrevin Feb 7, 2025
a333ffc
Working on Workflow tools
delarosatrevin Feb 11, 2025
122ddc4
More on Workflow
delarosatrevin Feb 12, 2025
1e113fb
Moved log function and more on Workflow
delarosatrevin Feb 12, 2025
b2b868a
Minor fixes in Workflow jobIndex
delarosatrevin Feb 13, 2025
c938f7d
Moved Acqusition and RelionStar to metadata
delarosatrevin Feb 14, 2025
b244b8a
Added function Job.hasOutput
delarosatrevin Feb 14, 2025
cdfa949
Added Timer to Batch
delarosatrevin Feb 17, 2025
41ab96d
Added Mdoc.glob and fixed circular import
delarosatrevin Feb 18, 2025
11643ed
Added more funcs to Batch and new class MdocBatchManager
delarosatrevin Feb 18, 2025
064744c
Added table for tiltseries
delarosatrevin Feb 18, 2025
eefb14f
Added global table for tiltseries
delarosatrevin Feb 18, 2025
edad74c
Added Path.rsync method
delarosatrevin Feb 20, 2025
525e6df
Fixed typo
delarosatrevin Feb 21, 2025
c7b30bf
Fixed typos
delarosatrevin Feb 24, 2025
7f6a751
Minor updates
delarosatrevin Feb 26, 2025
d0078f4
Added flush to log function
delarosatrevin Feb 28, 2025
92fc202
Added Vars helper class
delarosatrevin Mar 1, 2025
49c2af2
emt-image utility and added fractions.mrc as movie suffix
delarosatrevin Mar 3, 2025
8c121bb
Added some utilities
delarosatrevin Mar 4, 2025
99be009
Allow to avoid creation of batch folder
delarosatrevin Mar 9, 2025
ee190c3
Fixed bug in getTable with non-existing table
delarosatrevin Mar 10, 2025
1c428bb
Allow imageId in micrograph_table
delarosatrevin Mar 12, 2025
f97daaf
Added Timer.parse_timedelta function
delarosatrevin Mar 12, 2025
c4d59b5
Moved dump function to FolderManager
delarosatrevin Mar 16, 2025
d8b183c
Check if input star file exists
delarosatrevin Mar 18, 2025
3803714
Updated some utils methods
delarosatrevin Mar 25, 2025
986ca87
Move utils methods
delarosatrevin Mar 25, 2025
6037d64
Merge branch 'devel' of github.com:3dem/emtools into devel
delarosatrevin Mar 25, 2025
7ae3bff
Allow additional options to rsync
delarosatrevin Apr 13, 2025
64d9633
Fixed error counting dirs
delarosatrevin Apr 13, 2025
ec7774e
Fixed import
delarosatrevin Apr 13, 2025
de02625
Added option to count movies
delarosatrevin Apr 13, 2025
ce4b2c7
Added method EPU.count_movies
delarosatrevin Apr 18, 2025
2f872da
Make Process fail for Rsync and allow to check transfer size
delarosatrevin May 5, 2025
34ebd7b
Added duplicates check to emt-star and rename function to FolderManager
delarosatrevin May 15, 2025
e6b2e89
Print all duplicates
delarosatrevin May 15, 2025
5274a16
Added helper methods to determine filetype based on extension
delarosatrevin May 20, 2025
b11656d
Allow dict as input
delarosatrevin Jun 20, 2025
33309cb
Implemented TS batch manager from star input
delarosatrevin Jun 20, 2025
3e5f37a
Added utility functions to Batch and FolderManager
delarosatrevin Jun 30, 2025
fd46e50
working on a gpu monitor tool
delarosatrevin Jul 12, 2025
6e08731
Added WarpXml helper class
delarosatrevin Jul 16, 2025
b55f6f0
Added Batch.copy method
delarosatrevin Jul 22, 2025
b5f3db2
added dose properties to Acquisition class
delarosatrevin Jul 24, 2025
17dfd18
allow string arguments
delarosatrevin Aug 7, 2025
1e56134
emt-ps to catch program name in args
delarosatrevin Aug 22, 2025
285e839
Improved Mdocs batch manager
delarosatrevin Aug 23, 2025
b4a940c
Added debug shortcut
delarosatrevin Aug 25, 2025
bcda193
Allow to pass basename to link
delarosatrevin Sep 2, 2025
8a956ce
Fix when there is no last file
delarosatrevin Sep 4, 2025
ba7cee0
Some utils
delarosatrevin Sep 30, 2025
957449e
Add rsync extra args at the end
delarosatrevin Nov 5, 2025
e5038df
Fixed bug about new option computeFormat when writing StarFile tables
delarosatrevin Nov 10, 2025
bb00d65
Method to clear a folder
delarosatrevin Nov 11, 2025
40e240d
Added methods to read/write job.star and fixed bug in pipeline.star
delarosatrevin Nov 11, 2025
ee42714
Allow default to Workflow.getJob and added deleteJob
delarosatrevin Nov 11, 2025
3084574
Allow left alignment in format and fixed boolean values
delarosatrevin Nov 12, 2025
f44430d
Removed some debug print
delarosatrevin Nov 12, 2025
25a8a26
Added method to worklflow Job.getOutput
delarosatrevin Nov 13, 2025
2d67b09
Added argument to write timeStamp
delarosatrevin Nov 13, 2025
87fc681
Added missing requirement
delarosatrevin Nov 16, 2025
905d47d
Do not check stat if file does not exist
delarosatrevin Dec 11, 2025
c2c5553
Added .eer as tiff image
delarosatrevin Dec 16, 2025
c102a8f
Added Table.cloneColumns method to create empty table using columns f…
delarosatrevin Dec 22, 2025
ae84c64
Fixed double extension in .mdoc files
delarosatrevin Jan 24, 2026
fe1c243
Fixed handling of Relion's bool values
delarosatrevin Jan 24, 2026
02aaadc
Added method to validate missing data
delarosatrevin Jan 27, 2026
7941ba2
Updated Workflow class to use dict instead of lists
delarosatrevin Feb 1, 2026
2ed7f22
Added method to clear inputs and more text extensions
delarosatrevin Feb 2, 2026
44b6966
Added subset method to Args
delarosatrevin Feb 3, 2026
533a906
Added method to parse timedelta
delarosatrevin Feb 6, 2026
b2a93cc
Add folder scanning and JSON snapshot functionality
delarosatrevin Feb 11, 2026
699f173
added new textfiles extensions
delarosatrevin Feb 11, 2026
0bf8762
Added utility methods
delarosatrevin Feb 11, 2026
61e36b1
Added Thumbnail.Preview class and more file extensions
delarosatrevin Feb 23, 2026
cf3f1b1
Added helper class WarpPopulation
delarosatrevin Mar 3, 2026
a515cd3
Added a method to get starfile tables as dict
delarosatrevin Mar 31, 2026
a1b00a3
Added extra validation for setting path in FolderManager
delarosatrevin Apr 23, 2026
eaf9928
Added --relink option and fixed old options
delarosatrevin Apr 27, 2026
0e6275c
Added support for inverted_booleans and possible filters
delarosatrevin May 4, 2026
ddf4874
Fixed bug for possible checking
delarosatrevin May 5, 2026
595f468
Added compute hash and fixed rsync parsing for Mac
delarosatrevin May 22, 2026
f1163a8
Getting Species info from Warp output
delarosatrevin May 22, 2026
f246b88
Handling ending slash in jobids for relion pipeline
delarosatrevin Jun 13, 2026
376e969
Added relion tomo alignment constants
delarosatrevin Jun 23, 2026
53721db
Added more text file extensions
delarosatrevin Jun 26, 2026
e9d7452
Added option to print some STAR file columns
delarosatrevin Jun 26, 2026
a087545
Merge pull request #12 from 3dem/relion5-compat
delarosatrevin Jun 26, 2026
a02e260
TsStarBatchManager now also reads tsMdoc and is passed to the batch
Jul 6, 2026
54e36d4
Merge branch 'devel' of https://github.com/3dem/emtools into devel
Jul 6, 2026
d3a9215
Added binary_boolean filter to get 0/1 for boolean params
delarosatrevin Jul 6, 2026
9c494b3
Condensed in elif
delarosatrevin Jul 6, 2026
94a3569
Args subset function modified with multiple values
Jul 7, 2026
aadfed9
Added Job.getInput method and renamed label to rlnTomoMdocFile
delarosatrevin Jul 7, 2026
3aa3a31
Merge branch 'devel' of github.com:3dem/emtools into devel
delarosatrevin Jul 7, 2026
a471a37
Using regex to detect args with - or --
delarosatrevin Jul 7, 2026
3fd53ff
Added mdoc as text file
delarosatrevin Jul 7, 2026
e10db1d
Some Relion tomo colums refactoring
delarosatrevin Jul 8, 2026
2ffde73
Pass the whole row as a dict
delarosatrevin Jul 15, 2026
9678161
Added some more text extension
delarosatrevin Jul 17, 2026
1f6485c
Mapped unknown status in Relion to 'Saved'
delarosatrevin Jul 23, 2026
585b57e
Added Imod util class and methods to read .tlt and .xf files
delarosatrevin Jul 23, 2026
fb2fa49
Write the #order for starfiles and allow table from list of rows
delarosatrevin Jul 23, 2026
7025d8a
Allow to sort zsections by the DateTime value
delarosatrevin Jul 24, 2026
4340bed
Added methods to clearOutputs
delarosatrevin Jul 24, 2026
28caa8b
Added alignment conversion from Imod to Relion and fixed comment spac…
delarosatrevin Jul 27, 2026
a481153
Added some tomo functions to RelionStar
delarosatrevin Jul 28, 2026
6888403
Added helper methods
delarosatrevin Jul 29, 2026
ff12028
Added method to getAcquisition from STAR file
delarosatrevin Aug 5, 2026
4f390d5
Bumped version to 0.2.0-rc260805
delarosatrevin Aug 5, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion emtools/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@
# *
# **************************************************************************

__version__ = '0.1.3'
__version__ = '0.2.0-rc260805'

16 changes: 3 additions & 13 deletions emtools/image/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# **************************************************************************
# *
# * Authors: J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
# *
# * [1] SciLifeLab, Stockholm University
# * Authors: J.M. de la Rosa Trevin (delarosatrevin@gmail.com)
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
Expand All @@ -14,18 +12,10 @@
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307 USA
# *
# * All comments concerning this program package may be sent to the
# * e-mail address 'delarosatrevin@scilifelab.se'
# *
# **************************************************************************


from .thumbnail import Thumbnail
from .thumbnail import Thumbnail, Image


__all__ = [Thumbnail]
__all__ = ["Thumbnail", "Image"]
32 changes: 32 additions & 0 deletions emtools/image/__main__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# **************************************************************************
# *
# * Authors: J.M. de la Rosa Trevin (delarosatrevin@gmail.com)
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
# * the Free Software Foundation; either version 3 of the License, or
# * (at your option) any later version.
# *
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# * GNU General Public License for more details.
# *
# **************************************************************************

import argparse
from .thumbnail import Image


def main():
p = argparse.ArgumentParser()
p.add_argument('path', metavar="IMAGE_PATH",
help="Image path")

args = p.parse_args()

print(Image.get_dimensions(args.path))


if __name__ == '__main__':
main()
113 changes: 92 additions & 21 deletions emtools/image/thumbnail.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# **************************************************************************
# *
# * Authors: J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
# *
# * [1] SciLifeLab, Stockholm University
# * Authors: J.M. de la Rosa Trevin (delarosatrevin@gmail.com)
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
Expand All @@ -14,22 +12,19 @@
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307 USA
# *
# * All comments concerning this program package may be sent to the
# * e-mail address 'delarosatrevin@scilifelab.se'
# *
# **************************************************************************

from doctest import OutputChecker
import io
import numpy as np
import base64
import mrcfile
import tifffile

from PIL import Image, ImageOps, ImageFilter
import PIL
from PIL import Image

from emtools.utils import Path, Pretty


class Thumbnail:
Expand All @@ -49,7 +44,6 @@ def __init__(self, **kwargs):
self.min_max = kwargs.get('min_max', None)
self.std_threshold = kwargs.get('std_threshold', 0)


def __format(self, pil_img):
format = self.output_format

Expand Down Expand Up @@ -79,18 +73,18 @@ def from_pil(self, pil_img):
self.scale = scale

if self.contrast_factor is not None:
pil_img = ImageOps.autocontrast(pil_img, cutoff=self.contrast_factor)
pil_img = PIL.ImageOps.autocontrast(pil_img, cutoff=self.contrast_factor)

if self.gaussian_radius is not None:
pil_img = pil_img.filter(ImageFilter.GaussianBlur(radius=self.gaussian_radius))
pil_img = pil_img.filter(PIL.ImageFilter.GaussianBlur(radius=self.gaussian_radius))

return self.__format(pil_img)

def from_path(self, path):
""" Read the image path as a PIL image and encode it as base64.
"""
try:
img = Image.open(path)
img = PIL.Image.open(path)
encoded = self.from_pil(img)
img.close()
except:
Expand Down Expand Up @@ -121,7 +115,7 @@ def from_array(self, imageArray):

im255 = ((array - iMin) / (iMax - iMin) * 255).astype(np.uint8)

pil_img = Image.fromarray(im255)
pil_img = PIL.Image.fromarray(im255)

return self.from_pil(pil_img)

Expand All @@ -143,8 +137,8 @@ def from_mrc(self, mrc_path):

@staticmethod
def Micrograph(**kwargs):
""" Shortcut method with presets for Micrograph thumbail.
All settings can be overwriten with kwargs.
""" Shortcut method with presets for Micrograph thumbnail.
All settings can be overwritten with kwargs.
"""
defaults = {
'output_format': 'base64',
Expand All @@ -157,8 +151,8 @@ def Micrograph(**kwargs):

@staticmethod
def Psd(**kwargs):
""" Shortcut method with presets for PSD thumbails.
All settings can be overwriten with kwargs.
""" Shortcut method with presets for PSD thumbnails.
All settings can be overwritten with kwargs.
"""
defaults = {
'output_format': 'base64',
Expand All @@ -168,3 +162,80 @@ def Psd(**kwargs):
defaults.update(kwargs)
return Thumbnail(**defaults)

@staticmethod
def Preview(imagePath, **kwargs):
imageLower = imagePath.lower()
thumb = Thumbnail.Micrograph(max_size=(256, 256))

if not (Path.isImage(imagePath) or Path.isEmImage(imagePath)):
raise Exception("Can not generate preview for: %s" % imagePath)

if Path.isImage(imagePath):
return thumb.from_path(imagePath)

if imageLower.endswith('.mrc'):
dims = Image.get_dimensions(imagePath)
mrc = mrcfile.open(imagePath, permissive=True)
thumb = Thumbnail.Micrograph()
if len(dims) == 2:
array = mrc.data
elif len(dims) == 3:
x, y, z = dims
if mrc.is_volume() or (x == y and y == z):
thumb = Thumbnail(max_size=(256, 256), output_format='base64')
iMax = mrc.data.max() # min(imean + 10 * isd, imageArray.max())
iMin = mrc.data.min() # max(imean - 10 * isd, imageArray.min())
im255 = ((mrc.data - iMin) / (iMax - iMin) * 255).astype(np.uint8)

# 1. Setup
ximg = PIL.Image.fromarray(im255[:, :, x // 2])
yimg = PIL.Image.fromarray(im255[:, y // 2, :])
zimg = PIL.Image.fromarray(im255[z // 2, :, :])

xw, xh = ximg.size
yw, yh = yimg.size
zw, zh = zimg.size

pad = 2 # The thickness of the dark gray lines/borders

# 2. Calculate canvas size for a full grid with outer borders
# Total Width = (2 * image width) + (3 * padding for left, middle, right)
canvas_w = (xw + yw) + (3 * pad)
canvas_h = (xh + zh) + (3 * pad)

# Create canvas with a white background (matching your image)
bg_color = (256, 256, 256)
montage = PIL.Image.new('RGB', (canvas_w, canvas_h), bg_color)

# Top-Left: x slice
montage.paste(ximg, (pad, pad))
# Bottom-Left: z slice
montage.paste(zimg, (pad, xh + 2 * pad))
# Bottom-Right: y slice
montage.paste(yimg, (xw + 2 * pad, xh + 2 * pad))

return thumb.from_pil(montage)
Pretty.dprint("Loading MRC volume: %s" % str(array.shape))
else:
array = mrc.data[z//2, :, :] # FIXME
Pretty.dprint("Loading MRC 2D: %s" % str(array.shape))
return thumb.from_array(array)
else:
raise Exception("Invalid dimensions: %s" % dims)


class Image:
@staticmethod
def get_dimensions(imagePath):
imageLower = imagePath.lower()
if imageLower.endswith('.mrc') or imageLower.endswith('.mrcs'):
with mrcfile.open(imagePath) as mrc:
return mrc.data.shape[::-1] # in reverse order
elif (imageLower.endswith('.tif') or
imageLower.endswith('.tiff') or
imageLower.endswith('.eer') or
imageLower.endswith('.gain')):
with tifffile.TiffFile(imagePath) as tif:
n = len(tif.pages)
y, x = tif.pages[0].shape
return (x, y, n) if n > 1 else (x, y)
10 changes: 7 additions & 3 deletions emtools/jobs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@
# *
# **************************************************************************

from .pipeline import Pipeline, ProcessingPipeline
from .batch_manager import BatchManager
from .pipeline import Pipeline
from .batch_manager import (Args, Batch, Vars,
BatchManager, MdocBatchManager, TsStarBatchManager)
from .workflow import Workflow

__all__ = ["Pipeline", "BatchManager", "ProcessingPipeline"]
__all__ = ["Pipeline", "Workflow",
"Args", "Vars",
"Batch", "BatchManager", "MdocBatchManager", "TsStarBatchManager"]
Loading
Loading