From a0504a896d7b8abb49efbeadfab7be445a3db5a1 Mon Sep 17 00:00:00 2001 From: miguelhroca Date: Tue, 5 Dec 2023 14:15:13 +0100 Subject: [PATCH 1/5] create branch --- .../protocols/protocol_monitor_summary.py | 159 +++++++++++++++++- emfacilities/protocols/report_html.py | 142 +++++++++++++++- 2 files changed, 294 insertions(+), 7 deletions(-) diff --git a/emfacilities/protocols/protocol_monitor_summary.py b/emfacilities/protocols/protocol_monitor_summary.py index 92e1a428e..2be1e935b 100644 --- a/emfacilities/protocols/protocol_monitor_summary.py +++ b/emfacilities/protocols/protocol_monitor_summary.py @@ -30,7 +30,7 @@ import pyworkflow.protocol.params as params from pyworkflow import VERSION_1_1 -from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies +from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies, ProtParticlePicking #protocol_particles_picking from pwem import Domain import subprocess @@ -41,6 +41,18 @@ from .protocol_monitor_movie_gain import MonitorMovieGain from .protocol_monitor_system import MonitorSystem +from pwem import emlib, Domain +from pwem.protocols import EMProtocol +from pwem.objects import Class2D, Class3D, Image, CTFModel, Volume, Micrograph, Movie, Particle, SetOfCoordinates +from pyworkflow.protocol import params +from pyworkflow.object import String, Set +import pyworkflow.utils as pwutils +from .. import Plugin +from pyworkflow.project import config +from PIL import Image as ImagePIL +from PIL import ImageDraw +import subprocess + class ProtMonitorSummary(ProtMonitor): """ Provide some summary of the basic steps of the Scipion-Box: @@ -135,7 +147,6 @@ def _defineParams(self, form): form.addSection('Mail settings') ProtMonitor._sendMailParams(self, form) - form.addSection('HTML Report') form.addParam("doInflux", params.BooleanParam, @@ -146,7 +157,7 @@ def _defineParams(self, form): label="Publish command", help="Specify a command to publish the template. " "You can use the special token %(REPORT_FOLDER)s " - "that will be replaced with the report folder. " + "that will be replaced with the report folder." "For example: \n" "rsync -avL %(REPORT_FOLDER)s " "scipion@webserver:public_html/") @@ -217,6 +228,12 @@ def createReportDir(self): pathRepoSummary.write("HTML path to summary: " + self.reportPath) pathRepoSummary.close() + def _getPickingProtocol(self): + for protPointer in self.inputProtocols: + prot = protPointer.get() + if isinstance(prot, ProtParticlePicking): + return prot + return None def _getAlignProtocol(self): for protPointer in self.inputProtocols: @@ -225,6 +242,71 @@ def _getAlignProtocol(self): return prot return None + """def _plotMicroParticle(self , output): + self.outputName = output.getObjName() + outputDict = {} + outputDict[self.OUTPUT_NAME] = output.getObjName() + outputDict[self.OUTPUT_TYPE] = output.getClassName() + + items = [] + + # If output is a Set get a list with all items + if isinstance(output, Set): + outputDict[self.OUTPUT_SIZE] = output.getSize() + count = 0 + if isinstance(output, SetOfCoordinates): + coordinatesDict = {} + for micrograph in output.getMicrographs(): # get the first three micrographs + count += 1 + # apply a low pass filter + args = " -i %s -o %s --fourier low_pass %f" % (micrograph.getLocation()[1], self._getTmpPath(os.path.basename(micrograph.getFileName())), 0.05) + getEnviron = Domain.importFromPlugin('xmipp3', 'Plugin', doRaise=True).getEnviron + self.runJob('xmipp_transform_filter', args, env=getEnviron()) + # save jpg + repPath = self.getTopLevelPath(self.DIR_IMAGES, '%s_%s' % (self.outputName, pwutils.replaceBaseExt(micrograph.getFileName(), 'jpg'))) + self._ih.convert(self._getTmpPath(os.path.basename(micrograph.getFileName())), os.path.join(self.getProject().path, repPath)) + coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} + + items.append({self.ITEM_REPRESENTATION: repPath}) + if count == 3: break; + + for coordinate in output: # for each micrograph, get its coordinates + if coordinate.getMicName() in coordinatesDict: + coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) + + for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs + if 'coords' in values: + image = ImagePIL.open(values['path']).convert('RGB') + W_mic = values['Xdim'] + H_mic = values['Ydim'] + W_jpg, H_jpg = image.size + draw = ImageDraw.Draw(image) + r = W_jpg / 256 + for coord in values['coords']: + x = coord[0] * (W_jpg / W_mic) + y = coord[1] * (H_jpg / H_mic) + draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) + image.save(values['path'], quality=95) + print("image saved") + + else: + for item in output.iterItems(): + itemDict = self.getItemDict(item) + items.append(itemDict) + count += 1 + # In some types get only a limited number of items + if (isinstance(item, Micrograph) or isinstance(item, Movie) or isinstance(item, CTFModel)) and count == 3: break; + if isinstance(item, Particle) and count == 15: break; + + # If it is a single object then only one item is present + else: + items.append(self.getItemDict(output)) + + outputDict[self.OUTPUT_ITEMS] = items + + return outputDict""" + + def _getCtfProtocol(self): for protPointer in self.inputProtocols: prot = protPointer.get() @@ -305,8 +387,9 @@ def createSystemMonitor(self): doDiskIO=self.doDiskIO.get(), nif=MonitorSystem.getNifsNameList()[ self.netInterfaces.get()]) - + return sysMon + def getReportPath(self): return self.reportPath @@ -316,6 +399,8 @@ def createHtmlReport(self, ctfMonitor=None, sysMonitor=None, ctfMonitor = ctfMonitor or self.createCtfMonitor() sysMonitor = sysMonitor or self.createSystemMonitor() movieGainMonitor = movieGainMonitor or self.createMovieGainMonitor() + + #self._plotMicroParticle(self) self.createReportDir() if self.doInflux: htmlReport = ReportInflux(self, ctfMonitor, sysMonitor, movieGainMonitor, @@ -327,7 +412,73 @@ def createHtmlReport(self, ctfMonitor=None, sysMonitor=None, refreshSecs=self.samplingInterval.get()) htmlReport.setUp() + return htmlReport + + """def getMicroGraphs(self): + self.outputName = output.getObjName() + outputDict = {} + outputDict[self.OUTPUT_NAME] = output.getObjName() + outputDict[self.OUTPUT_TYPE] = output.getClassName() + + items = [] + + # If output is a Set get a list with all items + if isinstance(output, Set): + outputDict[self.OUTPUT_SIZE] = output.getSize() + count = 0 + if isinstance(output, SetOfCoordinates): + coordinatesDict = {} + for micrograph in output.getMicrographs(): # get the first three micrographs + count += 1 + # apply a low pass filter + args = " -i %s -o %s --fourier low_pass %f" % (micrograph.getLocation()[1], self._getTmpPath(os.path.basename(micrograph.getFileName())), 0.05) + getEnviron = Domain.importFromPlugin('xmipp3', 'Plugin', doRaise=True).getEnviron + self.runJob('xmipp_transform_filter', args, env=getEnviron()) + # save jpg + repPath = self.getTopLevelPath(self.DIR_IMAGES, '%s_%s' % (self.outputName, pwutils.replaceBaseExt(micrograph.getFileName(), 'jpg'))) + self._ih.convert(self._getTmpPath(os.path.basename(micrograph.getFileName())), os.path.join(self.getProject().path, repPath)) + coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} + + items.append({self.ITEM_REPRESENTATION: repPath}) + if count == 3: break; + + for coordinate in output: # for each micrograph, get its coordinates + if coordinate.getMicName() in coordinatesDict: + coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) + + for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs + if 'coords' in values: + image = ImagePIL.open(values['path']).convert('RGB') + W_mic = values['Xdim'] + H_mic = values['Ydim'] + W_jpg, H_jpg = image.size + draw = ImageDraw.Draw(image) + r = W_jpg / 256 + for coord in values['coords']: + x = coord[0] * (W_jpg / W_mic) + y = coord[1] * (H_jpg / H_mic) + draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) + image.save(values['path'], quality=95) + + else: + for item in output.iterItems(): + itemDict = self.getItemDict(item) + items.append(itemDict) + count += 1 + # In some types get only a limited number of items + if (isinstance(item, Micrograph) or isinstance(item, Movie) or isinstance(item, CTFModel)) and count == 3: break; + if isinstance(item, Particle) and count == 15: break; + + # If it is a single object then only one item is present + else: + items.append(self.getItemDict(output)) + + outputDict[self.OUTPUT_ITEMS] = items + + return outputDict""" + + def _summary(self): summary = [] pathRepoSummary = self._getPath("pathRepo.txt") diff --git a/emfacilities/protocols/report_html.py b/emfacilities/protocols/report_html.py index d6e82434c..8c0e2716b 100644 --- a/emfacilities/protocols/report_html.py +++ b/emfacilities/protocols/report_html.py @@ -33,6 +33,8 @@ import subprocess import multiprocessing from datetime import datetime +from PIL import Image as ImagePIL +from PIL import ImageDraw from statistics import median, mean from pyworkflow.protocol import getUpdatedProtocol @@ -58,6 +60,7 @@ MIC_ID = 'micId' DEFOCUS_HIST_BIN_WIDTH = 0.5 RESOLUTION_HIST_BIN_WIDTH = 0.5 +COORD = [] class ReportHtml: @@ -69,6 +72,7 @@ def __init__(self, protocol, ctfMonitor, sysMonitor, movieGainMonitor, publishCm self.protocol = protocol self.ctfProtocol = protocol._getCtfProtocol() self.alignProtocol = protocol._getAlignProtocol() + self.picking = protocol._getPickingProtocol() self.micThumbSymlinks = False self.reportPath = protocol.reportPath self.reportDir = protocol.reportDir @@ -94,10 +98,12 @@ def __init__(self, protocol, ctfMonitor, sysMonitor, movieGainMonitor, publishCm MIC_PATH: [], SHIFT_PATH: [], PSD_PATH: [], - MIC_ID: []} - + MIC_ID: []}#, + #COORD: []} + self.coordSet = [] # Get the html template to be used, by default use the one # in scipion/config/templates + print("inside template") self.template = self._getHTMLTemplatePath() self.publishCmd = publishCmd @@ -162,6 +168,30 @@ def setUp(self): and self.alignProtocol._doComputeMicThumbnail()): self.micThumbSymlinks = True + def getCoordset(pickingprotocol): + # TODO get this output names from Protocol constants + print("inside function78") + if hasattr(pickingprotocol, 'outputCoordinates'): + print("inside function77") + return pickingprotocol.outputCoordinates + + elif hasattr(pickingprotocol, 'outputCoordinates'): + print("inside function80") + return pickingprotocol.outputCoordinates + else: + print("inside function81") + return None + + def getboxsice(pickingprotocol): + # TODO get this output names from Protocol constants + if hasattr(pickingprotocol, 'boxsize'): + return pickingprotocol.boxsize + elif hasattr(pickingprotocol, 'boxsize'): + return pickingprotocol.boxsize + else: + return None + + def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='jpg', micIdSet=None): """Adds to self.thumbPaths the paths to the report thumbnails that come from the alignment and/or ctf protocol. @@ -213,6 +243,9 @@ def getMicSet(alignedProt): return else: return + + if self.picking is not None: + updatedProt = getUpdatedProtocol(self.picking) for micId in micIdSet[thumbsDone:]: mic = outputSet[micId] @@ -263,6 +296,106 @@ def getMicPSDPath(mic): if PSD_PATH in self.thumbPaths: self.thumbPaths.pop(PSD_PATH, None) + def _plotMicroParticle(self): + self.outputName = output.getObjName() + outputDict = {} + outputDict[self.OUTPUT_NAME] = output.getObjName() + outputDict[self.OUTPUT_TYPE] = output.getClassName() + + items = [] + + # If output is a Set get a list with all items + if isinstance(output, Set): + outputDict[self.OUTPUT_SIZE] = output.getSize() + count = 0 + if isinstance(output, SetOfCoordinates): + coordinatesDict = {} + for micrograph in output.getMicrographs(): # get the first three micrographs + count += 1 + # apply a low pass filter + args = " -i %s -o %s --fourier low_pass %f" % (micrograph.getLocation()[1], self._getTmpPath(os.path.basename(micrograph.getFileName())), 0.05) + getEnviron = Domain.importFromPlugin('xmipp3', 'Plugin', doRaise=True).getEnviron + self.runJob('xmipp_transform_filter', args, env=getEnviron()) + # save jpg + repPath = self.getTopLevelPath(self.DIR_IMAGES, '%s_%s' % (self.outputName, pwutils.replaceBaseExt(micrograph.getFileName(), 'jpg'))) + self._ih.convert(self._getTmpPath(os.path.basename(micrograph.getFileName())), os.path.join(self.getProject().path, repPath)) + coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} + + items.append({self.ITEM_REPRESENTATION: repPath}) + if count == 3: break; + + for coordinate in output: # for each micrograph, get its coordinates + if coordinate.getMicName() in coordinatesDict: + coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) + + for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs + if 'coords' in values: + image = ImagePIL.open(values['path']).convert('RGB') + W_mic = values['Xdim'] + H_mic = values['Ydim'] + W_jpg, H_jpg = image.size + draw = ImageDraw.Draw(image) + r = W_jpg / 256 + for coord in values['coords']: + x = coord[0] * (W_jpg / W_mic) + y = coord[1] * (H_jpg / H_mic) + draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) + image.save(values['path'], quality=95) + print("image saved") + + else: + for item in output.iterItems(): + itemDict = self.getItemDict(item) + items.append(itemDict) + count += 1 + # In some types get only a limited number of items + if (isinstance(item, Micrograph) or isinstance(item, Movie) or isinstance(item, CTFModel)) and count == 3: break; + if isinstance(item, Particle) and count == 15: break; + + # If it is a single object then only one item is present + else: + items.append(self.getItemDict(output)) + + outputDict[self.OUTPUT_ITEMS] = items + + return outputDict + def plotParticlePicking(self): + + print("inside function") + coord=self.getCoordset(self.picking) + + boxsize=self.getboxsice(self.picking) + + #mic=self.getCoordset().getMicrographs() + + print("inside function2") + coordinatesDict = {} + print("inside function34") + print("cooroddoeodoasd",coord) + for coordinate in coord: # for each micrograph, get its coordinates + print("inside function6") + if coordinate.getMicName() in coordinatesDict: + print("coordenadas dentro") + coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) + print("primera coordenada",coord) + print("inside function4") + + """for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs + if 'coords' in values: + image = ImagePIL.open(values['path']).convert('RGB') + W_mic = values['Xdim'] + H_mic = values['Ydim'] + W_jpg, H_jpg = image.size + draw = ImageDraw.Draw(image) + r = W_jpg / 256 + for coord in values['coords']: + x = coord[0] * (W_jpg / W_mic) + y = coord[1] * (H_jpg / H_mic) + draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) + image.save(values['path'], quality=95) + print("image saved")""" + + def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): """ Function to generate thumbnails for the report. Uses data from self.thumbPaths. @@ -522,12 +655,15 @@ def generate(self, finished): thumbsLoading = numMics - self.thumbsReady for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]: if k in self.thumbPaths: - data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading + data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading ## aqui va el mic con las coord data[MIC_ID] = self.thumbPaths[MIC_ID] reportFinished = self.thumbsReady == numMics + if data: + self.plotParticlePicking() + def convert(o): if isinstance(o, np.int64): return int(o) raise TypeError From 627f63dfff3509396459e892c1da64ea2ff024c0 Mon Sep 17 00:00:00 2001 From: miguelhroca Date: Tue, 12 Dec 2023 00:51:00 +0100 Subject: [PATCH 2/5] picking drawing --- emfacilities/protocols/report_html.py | 93 ++++++++++++++++----------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/emfacilities/protocols/report_html.py b/emfacilities/protocols/report_html.py index 8c0e2716b..facafd762 100644 --- a/emfacilities/protocols/report_html.py +++ b/emfacilities/protocols/report_html.py @@ -50,6 +50,7 @@ MIC_PATH = 'imgMicPath' PSD_PATH = 'imgPsdPath' SHIFT_PATH = 'imgShiftPath' +PICK_PATH = 'imgPickPath' # These constants are the name of the folders where thumbnails # for the html report will be stored. They are also the keys to # used in the execution.summary.template.html to read data (where @@ -57,6 +58,7 @@ MIC_THUMBS = 'imgMicThumbs' PSD_THUMBS = 'imgPsdThumbs' SHIFT_THUMBS = 'imgShiftThumbs' +PICK_THUMBS = 'imgPickThumbs' MIC_ID = 'micId' DEFOCUS_HIST_BIN_WIDTH = 0.5 RESOLUTION_HIST_BIN_WIDTH = 0.5 @@ -95,15 +97,16 @@ def __init__(self, protocol, ctfMonitor, sysMonitor, movieGainMonitor, publishCm self.thumbPaths = {MIC_THUMBS: [], PSD_THUMBS: [], SHIFT_THUMBS: [], + PICK_THUMBS:[], MIC_PATH: [], SHIFT_PATH: [], PSD_PATH: [], - MIC_ID: []}#, - #COORD: []} + PICK_PATH:[], + MIC_ID: [], + } self.coordSet = [] # Get the html template to be used, by default use the one # in scipion/config/templates - print("inside template") self.template = self._getHTMLTemplatePath() self.publishCmd = publishCmd @@ -168,26 +171,21 @@ def setUp(self): and self.alignProtocol._doComputeMicThumbnail()): self.micThumbSymlinks = True - def getCoordset(pickingprotocol): + def getCoordset(self): # TODO get this output names from Protocol constants - print("inside function78") - if hasattr(pickingprotocol, 'outputCoordinates'): - print("inside function77") - return pickingprotocol.outputCoordinates - - elif hasattr(pickingprotocol, 'outputCoordinates'): - print("inside function80") - return pickingprotocol.outputCoordinates + if hasattr(self.picking, 'outputCoordinates'): + return self.picking.outputCoordinates + elif hasattr(self.picking, 'outputCoordinates'): + return self.picking.outputCoordinates else: - print("inside function81") return None - def getboxsice(pickingprotocol): + def getboxsice(self): # TODO get this output names from Protocol constants - if hasattr(pickingprotocol, 'boxsize'): - return pickingprotocol.boxsize - elif hasattr(pickingprotocol, 'boxsize'): - return pickingprotocol.boxsize + if hasattr(self.picking, 'boxsize'): + return self.picking.boxsize + elif hasattr(self.picking, 'boxsize'): + return self.picking.boxsize else: return None @@ -259,6 +257,11 @@ def getMicSet(alignedProt): self.thumbPaths[MIC_PATH].append(srcMicFn) self.thumbPaths[MIC_THUMBS].append(micThumbFn) + if self.picking is not None: + micThumbFn = join(PICK_THUMBS, pwutils.replaceExt(basename(srcMicFn), ext)) + self.thumbPaths[PICK_PATH].append(srcMicFn) + self.thumbPaths[PICK_THUMBS].append(micThumbFn) + shiftPlot = (getattr(mic, 'plotCart', None) or getattr(mic, 'plotGlobal', None)) if shiftPlot is not None: shiftPath = "" if shiftPlot is None else abspath(shiftPlot.getFileName()) @@ -296,7 +299,8 @@ def getMicPSDPath(mic): if PSD_PATH in self.thumbPaths: self.thumbPaths.pop(PSD_PATH, None) - def _plotMicroParticle(self): + + """def _plotMicroParticle(self): self.outputName = output.getObjName() outputDict = {} outputDict[self.OUTPUT_NAME] = output.getObjName() @@ -320,6 +324,8 @@ def _plotMicroParticle(self): repPath = self.getTopLevelPath(self.DIR_IMAGES, '%s_%s' % (self.outputName, pwutils.replaceBaseExt(micrograph.getFileName(), 'jpg'))) self._ih.convert(self._getTmpPath(os.path.basename(micrograph.getFileName())), os.path.join(self.getProject().path, repPath)) coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} + + self._getExtraPath() items.append({self.ITEM_REPRESENTATION: repPath}) if count == 3: break; @@ -358,42 +364,50 @@ def _plotMicroParticle(self): outputDict[self.OUTPUT_ITEMS] = items - return outputDict + return outputDict""" + def plotParticlePicking(self): + """ + Function to plot 2D particle Picking + """ - print("inside function") - coord=self.getCoordset(self.picking) + coord=self.getCoordset() - boxsize=self.getboxsice(self.picking) + boxsize=self.getboxsice() - #mic=self.getCoordset().getMicrographs() - - print("inside function2") + mic=self.getCoordset().getMicrographs() coordinatesDict = {} - print("inside function34") - print("cooroddoeodoasd",coord) + + i=0 + numMics = len(self.thumbPaths[MIC_PATH]) + + for micrograph in mic: + + if i<=numMics: + repPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) + coordinatesDict[micrograph.getMicName()] = { 'path': repPath,'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} + i=i+1 + for coordinate in coord: # for each micrograph, get its coordinates - print("inside function6") if coordinate.getMicName() in coordinatesDict: - print("coordenadas dentro") coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) - print("primera coordenada",coord) - print("inside function4") - - """for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs + + for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs if 'coords' in values: image = ImagePIL.open(values['path']).convert('RGB') W_mic = values['Xdim'] H_mic = values['Ydim'] W_jpg, H_jpg = image.size draw = ImageDraw.Draw(image) - r = W_jpg / 256 + print("w_phge",W_jpg) + r = int(boxsize)/2 + border_color = (0, 255, 0) # Set the border color here for coord in values['coords']: x = coord[0] * (W_jpg / W_mic) y = coord[1] * (H_jpg / H_mic) - draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) + draw.ellipse((x - r, y - r, x + r, y + r),outline=border_color) image.save(values['path'], quality=95) - print("image saved")""" + print("image saved") def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): @@ -413,12 +427,17 @@ def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): print('Generating images for mic %d' % (i+1)) # mic thumbnails dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) + if not exists(dstImgPath): if self.micThumbSymlinks: pwutils.copyFile(self.thumbPaths[MIC_PATH][i], dstImgPath) else: ih.convert(self.thumbPaths[MIC_PATH][i], pwutils.replaceExt(dstImgPath, "jpg")) + + + + # shift plots if SHIFT_THUMBS in self.thumbPaths: dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i]) From 5be51c656a7eb5e6325c6ebf68e72e9c6739e96c Mon Sep 17 00:00:00 2001 From: miguelhroca Date: Tue, 30 Jan 2024 13:46:53 +0100 Subject: [PATCH 3/5] clean --- emfacilities/protocols/report_html.py | 69 --------------------------- 1 file changed, 69 deletions(-) diff --git a/emfacilities/protocols/report_html.py b/emfacilities/protocols/report_html.py index facafd762..2bc409e02 100644 --- a/emfacilities/protocols/report_html.py +++ b/emfacilities/protocols/report_html.py @@ -298,73 +298,6 @@ def getMicPSDPath(mic): self.thumbPaths.pop(PSD_THUMBS, None) if PSD_PATH in self.thumbPaths: self.thumbPaths.pop(PSD_PATH, None) - - - """def _plotMicroParticle(self): - self.outputName = output.getObjName() - outputDict = {} - outputDict[self.OUTPUT_NAME] = output.getObjName() - outputDict[self.OUTPUT_TYPE] = output.getClassName() - - items = [] - - # If output is a Set get a list with all items - if isinstance(output, Set): - outputDict[self.OUTPUT_SIZE] = output.getSize() - count = 0 - if isinstance(output, SetOfCoordinates): - coordinatesDict = {} - for micrograph in output.getMicrographs(): # get the first three micrographs - count += 1 - # apply a low pass filter - args = " -i %s -o %s --fourier low_pass %f" % (micrograph.getLocation()[1], self._getTmpPath(os.path.basename(micrograph.getFileName())), 0.05) - getEnviron = Domain.importFromPlugin('xmipp3', 'Plugin', doRaise=True).getEnviron - self.runJob('xmipp_transform_filter', args, env=getEnviron()) - # save jpg - repPath = self.getTopLevelPath(self.DIR_IMAGES, '%s_%s' % (self.outputName, pwutils.replaceBaseExt(micrograph.getFileName(), 'jpg'))) - self._ih.convert(self._getTmpPath(os.path.basename(micrograph.getFileName())), os.path.join(self.getProject().path, repPath)) - coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} - - self._getExtraPath() - - items.append({self.ITEM_REPRESENTATION: repPath}) - if count == 3: break; - - for coordinate in output: # for each micrograph, get its coordinates - if coordinate.getMicName() in coordinatesDict: - coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) - - for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs - if 'coords' in values: - image = ImagePIL.open(values['path']).convert('RGB') - W_mic = values['Xdim'] - H_mic = values['Ydim'] - W_jpg, H_jpg = image.size - draw = ImageDraw.Draw(image) - r = W_jpg / 256 - for coord in values['coords']: - x = coord[0] * (W_jpg / W_mic) - y = coord[1] * (H_jpg / H_mic) - draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) - image.save(values['path'], quality=95) - print("image saved") - - else: - for item in output.iterItems(): - itemDict = self.getItemDict(item) - items.append(itemDict) - count += 1 - # In some types get only a limited number of items - if (isinstance(item, Micrograph) or isinstance(item, Movie) or isinstance(item, CTFModel)) and count == 3: break; - if isinstance(item, Particle) and count == 15: break; - - # If it is a single object then only one item is present - else: - items.append(self.getItemDict(output)) - - outputDict[self.OUTPUT_ITEMS] = items - - return outputDict""" def plotParticlePicking(self): """ @@ -399,7 +332,6 @@ def plotParticlePicking(self): H_mic = values['Ydim'] W_jpg, H_jpg = image.size draw = ImageDraw.Draw(image) - print("w_phge",W_jpg) r = int(boxsize)/2 border_color = (0, 255, 0) # Set the border color here for coord in values['coords']: @@ -407,7 +339,6 @@ def plotParticlePicking(self): y = coord[1] * (H_jpg / H_mic) draw.ellipse((x - r, y - r, x + r, y + r),outline=border_color) image.save(values['path'], quality=95) - print("image saved") def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): From 3206e145916f3e0555016ed9943f154bc0a5405b Mon Sep 17 00:00:00 2001 From: miguelhroca Date: Mon, 12 Feb 2024 08:29:20 +0100 Subject: [PATCH 4/5] test done --- .../protocols/protocol_monitor_summary.py | 161 +++--------------- 1 file changed, 26 insertions(+), 135 deletions(-) diff --git a/emfacilities/protocols/protocol_monitor_summary.py b/emfacilities/protocols/protocol_monitor_summary.py index 2be1e935b..118898fbb 100644 --- a/emfacilities/protocols/protocol_monitor_summary.py +++ b/emfacilities/protocols/protocol_monitor_summary.py @@ -30,7 +30,7 @@ import pyworkflow.protocol.params as params from pyworkflow import VERSION_1_1 -from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies, ProtParticlePicking #protocol_particles_picking +from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies, ProtParticlePickingAuto #protocol_particles_picking from pwem import Domain import subprocess @@ -53,6 +53,13 @@ from PIL import ImageDraw import subprocess +from pyworkflow.protocol.constants import STATUS_NEW +from pyworkflow.protocol.params import PointerParam +from pyworkflow.utils.properties import Message + +from pwem.protocols import ProtProcessMovies +from pwem.objects import SetOfMicrographs, SetOfMovies + class ProtMonitorSummary(ProtMonitor): """ Provide some summary of the basic steps of the Scipion-Box: @@ -66,6 +73,7 @@ class ProtMonitorSummary(ProtMonitor): def __init__(self, **kwargs): ProtMonitor.__init__(self, **kwargs) + self.reportDir = '' self.reportPath = '' @@ -103,9 +111,6 @@ def _defineParams(self, form): label="Raise Alarm if astigmatism >", help="Raise alarm if astigmatism (defocusU-defocusV)is greater than given " "value") - - - form.addSection('System Monitor') form.addParam('cpuAlert', params.FloatParam, default=101, label="Raise Alarm if CPU > XX%", @@ -161,6 +166,8 @@ def _defineParams(self, form): "For example: \n" "rsync -avL %(REPORT_FOLDER)s " "scipion@webserver:public_html/") + + # --------------------------- INSERT steps functions --------------------- def _insertAllSteps(self): @@ -201,12 +208,24 @@ def stepAll(): # when sysmonitor done all protocols done sysMonitorFinished = sysMonitor.step() htmlFinished = reportHtml.generate(finished) + + + """def __init__2(self, **kwargs): + + ProtProcessMovies.__init__(self, **kwargs) + __init__2(self) + movie = self.inputMovies.get()[1].clone() + alignment = movie.getAlignment() + + # getShifts() returns the absolute shifts from a certain reference + shiftListX, shiftListY = alignment.getShifts() + print("lista de shitfts",shiftListX,"esta es la x")""" if sysMonitorFinished and htmlFinished: finished = True reportHtml.generate(finished) except Exception as ex: - print("An error happened:") + print("An error happened:",ex) import traceback traceback.print_exc() @@ -231,7 +250,8 @@ def createReportDir(self): def _getPickingProtocol(self): for protPointer in self.inputProtocols: prot = protPointer.get() - if isinstance(prot, ProtParticlePicking): + print(type(prot)) + if isinstance(prot, ProtParticlePickingAuto): return prot return None @@ -242,71 +262,6 @@ def _getAlignProtocol(self): return prot return None - """def _plotMicroParticle(self , output): - self.outputName = output.getObjName() - outputDict = {} - outputDict[self.OUTPUT_NAME] = output.getObjName() - outputDict[self.OUTPUT_TYPE] = output.getClassName() - - items = [] - - # If output is a Set get a list with all items - if isinstance(output, Set): - outputDict[self.OUTPUT_SIZE] = output.getSize() - count = 0 - if isinstance(output, SetOfCoordinates): - coordinatesDict = {} - for micrograph in output.getMicrographs(): # get the first three micrographs - count += 1 - # apply a low pass filter - args = " -i %s -o %s --fourier low_pass %f" % (micrograph.getLocation()[1], self._getTmpPath(os.path.basename(micrograph.getFileName())), 0.05) - getEnviron = Domain.importFromPlugin('xmipp3', 'Plugin', doRaise=True).getEnviron - self.runJob('xmipp_transform_filter', args, env=getEnviron()) - # save jpg - repPath = self.getTopLevelPath(self.DIR_IMAGES, '%s_%s' % (self.outputName, pwutils.replaceBaseExt(micrograph.getFileName(), 'jpg'))) - self._ih.convert(self._getTmpPath(os.path.basename(micrograph.getFileName())), os.path.join(self.getProject().path, repPath)) - coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} - - items.append({self.ITEM_REPRESENTATION: repPath}) - if count == 3: break; - - for coordinate in output: # for each micrograph, get its coordinates - if coordinate.getMicName() in coordinatesDict: - coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) - - for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs - if 'coords' in values: - image = ImagePIL.open(values['path']).convert('RGB') - W_mic = values['Xdim'] - H_mic = values['Ydim'] - W_jpg, H_jpg = image.size - draw = ImageDraw.Draw(image) - r = W_jpg / 256 - for coord in values['coords']: - x = coord[0] * (W_jpg / W_mic) - y = coord[1] * (H_jpg / H_mic) - draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) - image.save(values['path'], quality=95) - print("image saved") - - else: - for item in output.iterItems(): - itemDict = self.getItemDict(item) - items.append(itemDict) - count += 1 - # In some types get only a limited number of items - if (isinstance(item, Micrograph) or isinstance(item, Movie) or isinstance(item, CTFModel)) and count == 3: break; - if isinstance(item, Particle) and count == 15: break; - - # If it is a single object then only one item is present - else: - items.append(self.getItemDict(output)) - - outputDict[self.OUTPUT_ITEMS] = items - - return outputDict""" - - def _getCtfProtocol(self): for protPointer in self.inputProtocols: prot = protPointer.get() @@ -414,70 +369,6 @@ def createHtmlReport(self, ctfMonitor=None, sysMonitor=None, return htmlReport - - """def getMicroGraphs(self): - self.outputName = output.getObjName() - outputDict = {} - outputDict[self.OUTPUT_NAME] = output.getObjName() - outputDict[self.OUTPUT_TYPE] = output.getClassName() - - items = [] - - # If output is a Set get a list with all items - if isinstance(output, Set): - outputDict[self.OUTPUT_SIZE] = output.getSize() - count = 0 - if isinstance(output, SetOfCoordinates): - coordinatesDict = {} - for micrograph in output.getMicrographs(): # get the first three micrographs - count += 1 - # apply a low pass filter - args = " -i %s -o %s --fourier low_pass %f" % (micrograph.getLocation()[1], self._getTmpPath(os.path.basename(micrograph.getFileName())), 0.05) - getEnviron = Domain.importFromPlugin('xmipp3', 'Plugin', doRaise=True).getEnviron - self.runJob('xmipp_transform_filter', args, env=getEnviron()) - # save jpg - repPath = self.getTopLevelPath(self.DIR_IMAGES, '%s_%s' % (self.outputName, pwutils.replaceBaseExt(micrograph.getFileName(), 'jpg'))) - self._ih.convert(self._getTmpPath(os.path.basename(micrograph.getFileName())), os.path.join(self.getProject().path, repPath)) - coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} - - items.append({self.ITEM_REPRESENTATION: repPath}) - if count == 3: break; - - for coordinate in output: # for each micrograph, get its coordinates - if coordinate.getMicName() in coordinatesDict: - coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) - - for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs - if 'coords' in values: - image = ImagePIL.open(values['path']).convert('RGB') - W_mic = values['Xdim'] - H_mic = values['Ydim'] - W_jpg, H_jpg = image.size - draw = ImageDraw.Draw(image) - r = W_jpg / 256 - for coord in values['coords']: - x = coord[0] * (W_jpg / W_mic) - y = coord[1] * (H_jpg / H_mic) - draw.ellipse((x - r, y - r, x + r, y + r), fill=(0, 255, 0)) - image.save(values['path'], quality=95) - - else: - for item in output.iterItems(): - itemDict = self.getItemDict(item) - items.append(itemDict) - count += 1 - # In some types get only a limited number of items - if (isinstance(item, Micrograph) or isinstance(item, Movie) or isinstance(item, CTFModel)) and count == 3: break; - if isinstance(item, Particle) and count == 15: break; - - # If it is a single object then only one item is present - else: - items.append(self.getItemDict(output)) - - outputDict[self.OUTPUT_ITEMS] = items - - return outputDict""" - def _summary(self): summary = [] From 40e84e37e0a0053a0625e2bb96bd20d758ff9ed3 Mon Sep 17 00:00:00 2001 From: Daniel Marchan Date: Fri, 16 Feb 2024 10:13:41 +0000 Subject: [PATCH 5/5] cleaning code from imports and commented code --- .../protocols/protocol_monitor_summary.py | 42 +------------ emfacilities/protocols/report_html.py | 63 ++++++++----------- 2 files changed, 29 insertions(+), 76 deletions(-) diff --git a/emfacilities/protocols/protocol_monitor_summary.py b/emfacilities/protocols/protocol_monitor_summary.py index 118898fbb..387b899c4 100644 --- a/emfacilities/protocols/protocol_monitor_summary.py +++ b/emfacilities/protocols/protocol_monitor_summary.py @@ -30,7 +30,7 @@ import pyworkflow.protocol.params as params from pyworkflow import VERSION_1_1 -from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies, ProtParticlePickingAuto #protocol_particles_picking +from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies, ProtParticlePickingAuto from pwem import Domain import subprocess @@ -41,25 +41,6 @@ from .protocol_monitor_movie_gain import MonitorMovieGain from .protocol_monitor_system import MonitorSystem -from pwem import emlib, Domain -from pwem.protocols import EMProtocol -from pwem.objects import Class2D, Class3D, Image, CTFModel, Volume, Micrograph, Movie, Particle, SetOfCoordinates -from pyworkflow.protocol import params -from pyworkflow.object import String, Set -import pyworkflow.utils as pwutils -from .. import Plugin -from pyworkflow.project import config -from PIL import Image as ImagePIL -from PIL import ImageDraw -import subprocess - -from pyworkflow.protocol.constants import STATUS_NEW -from pyworkflow.protocol.params import PointerParam -from pyworkflow.utils.properties import Message - -from pwem.protocols import ProtProcessMovies -from pwem.objects import SetOfMicrographs, SetOfMovies - class ProtMonitorSummary(ProtMonitor): """ Provide some summary of the basic steps of the Scipion-Box: @@ -73,7 +54,6 @@ class ProtMonitorSummary(ProtMonitor): def __init__(self, **kwargs): ProtMonitor.__init__(self, **kwargs) - self.reportDir = '' self.reportPath = '' @@ -111,12 +91,12 @@ def _defineParams(self, form): label="Raise Alarm if astigmatism >", help="Raise alarm if astigmatism (defocusU-defocusV)is greater than given " "value") + form.addSection('System Monitor') form.addParam('cpuAlert', params.FloatParam, default=101, label="Raise Alarm if CPU > XX%", help="Raise alarm if memory allocated is greater " "than given percentage") - form.addParam('memAlert', params.FloatParam, default=101, label="Raise Alarm if Memory > XX%", help="Raise alarm if cpu allocated is greater " @@ -153,7 +133,6 @@ def _defineParams(self, form): form.addSection('Mail settings') ProtMonitor._sendMailParams(self, form) form.addSection('HTML Report') - form.addParam("doInflux", params.BooleanParam, label="use grafana/influx", default=False, @@ -166,8 +145,6 @@ def _defineParams(self, form): "For example: \n" "rsync -avL %(REPORT_FOLDER)s " "scipion@webserver:public_html/") - - # --------------------------- INSERT steps functions --------------------- def _insertAllSteps(self): @@ -209,23 +186,12 @@ def stepAll(): sysMonitorFinished = sysMonitor.step() htmlFinished = reportHtml.generate(finished) - - """def __init__2(self, **kwargs): - - ProtProcessMovies.__init__(self, **kwargs) - __init__2(self) - movie = self.inputMovies.get()[1].clone() - alignment = movie.getAlignment() - - # getShifts() returns the absolute shifts from a certain reference - shiftListX, shiftListY = alignment.getShifts() - print("lista de shitfts",shiftListX,"esta es la x")""" if sysMonitorFinished and htmlFinished: finished = True reportHtml.generate(finished) except Exception as ex: - print("An error happened:",ex) + print("An error happened:", ex) import traceback traceback.print_exc() @@ -355,7 +321,6 @@ def createHtmlReport(self, ctfMonitor=None, sysMonitor=None, sysMonitor = sysMonitor or self.createSystemMonitor() movieGainMonitor = movieGainMonitor or self.createMovieGainMonitor() - #self._plotMicroParticle(self) self.createReportDir() if self.doInflux: htmlReport = ReportInflux(self, ctfMonitor, sysMonitor, movieGainMonitor, @@ -367,7 +332,6 @@ def createHtmlReport(self, ctfMonitor=None, sysMonitor=None, refreshSecs=self.samplingInterval.get()) htmlReport.setUp() - return htmlReport def _summary(self): diff --git a/emfacilities/protocols/report_html.py b/emfacilities/protocols/report_html.py index 2bc409e02..8498d12eb 100644 --- a/emfacilities/protocols/report_html.py +++ b/emfacilities/protocols/report_html.py @@ -101,7 +101,7 @@ def __init__(self, protocol, ctfMonitor, sysMonitor, movieGainMonitor, publishCm MIC_PATH: [], SHIFT_PATH: [], PSD_PATH: [], - PICK_PATH:[], + PICK_PATH: [], MIC_ID: [], } self.coordSet = [] @@ -189,7 +189,6 @@ def getboxsice(self): else: return None - def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='jpg', micIdSet=None): """Adds to self.thumbPaths the paths to the report thumbnails that come from the alignment and/or ctf protocol. @@ -303,43 +302,38 @@ def plotParticlePicking(self): """ Function to plot 2D particle Picking """ - - coord=self.getCoordset() - - boxsize=self.getboxsice() - - mic=self.getCoordset().getMicrographs() + coord = self.getCoordset() + boxsize = self.getboxsice() + mic = self.getCoordset().getMicrographs() coordinatesDict = {} - - i=0 + i = 0 numMics = len(self.thumbPaths[MIC_PATH]) for micrograph in mic: - - if i<=numMics: - repPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) - coordinatesDict[micrograph.getMicName()] = { 'path': repPath,'Xdim': micrograph.getXDim(), 'Ydim': micrograph.getYDim()} - i=i+1 + if i <= numMics: + repPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) + coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), + 'Ydim': micrograph.getYDim()} + i = i+1 for coordinate in coord: # for each micrograph, get its coordinates - if coordinate.getMicName() in coordinatesDict: - coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) + if coordinate.getMicName() in coordinatesDict: + coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs - if 'coords' in values: - image = ImagePIL.open(values['path']).convert('RGB') - W_mic = values['Xdim'] - H_mic = values['Ydim'] - W_jpg, H_jpg = image.size - draw = ImageDraw.Draw(image) - r = int(boxsize)/2 - border_color = (0, 255, 0) # Set the border color here - for coord in values['coords']: - x = coord[0] * (W_jpg / W_mic) - y = coord[1] * (H_jpg / H_mic) - draw.ellipse((x - r, y - r, x + r, y + r),outline=border_color) - image.save(values['path'], quality=95) - + if 'coords' in values: + image = ImagePIL.open(values['path']).convert('RGB') + W_mic = values['Xdim'] + H_mic = values['Ydim'] + W_jpg, H_jpg = image.size + draw = ImageDraw.Draw(image) + r = int(boxsize)/2 + border_color = (0, 255, 0) # Set the border color here + for coord in values['coords']: + x = coord[0] * (W_jpg / W_mic) + y = coord[1] * (H_jpg / H_mic) + draw.ellipse((x - r, y - r, x + r, y + r), outline=border_color) + image.save(values['path'], quality=95) def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): """ Function to generate thumbnails for the report. Uses data from @@ -365,10 +359,6 @@ def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): else: ih.convert(self.thumbPaths[MIC_PATH][i], pwutils.replaceExt(dstImgPath, "jpg")) - - - - # shift plots if SHIFT_THUMBS in self.thumbPaths: dstImgPath = join(self.reportDir, self.thumbPaths[SHIFT_THUMBS][i]) @@ -470,7 +460,6 @@ def getTimeSeries(self, data): # Get timeStamp ts = data[TIME_STAMP] - timeSeries = dict() # Get phaseShift @@ -605,7 +594,7 @@ def generate(self, finished): thumbsLoading = numMics - self.thumbsReady for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]: if k in self.thumbPaths: - data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading ## aqui va el mic con las coord + data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading data[MIC_ID] = self.thumbPaths[MIC_ID]