diff --git a/emfacilities/protocols/protocol_monitor_summary.py b/emfacilities/protocols/protocol_monitor_summary.py index 92e1a428e..387b899c4 100644 --- a/emfacilities/protocols/protocol_monitor_summary.py +++ b/emfacilities/protocols/protocol_monitor_summary.py @@ -30,7 +30,7 @@ import pyworkflow.protocol.params as params from pyworkflow import VERSION_1_1 -from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies +from pwem.protocols import ProtCTFMicrographs, ProtAlignMovies, ProtParticlePickingAuto from pwem import Domain import subprocess @@ -92,14 +92,11 @@ def _defineParams(self, form): help="Raise alarm if astigmatism (defocusU-defocusV)is greater than given " "value") - - form.addSection('System Monitor') form.addParam('cpuAlert', params.FloatParam, default=101, label="Raise Alarm if CPU > XX%", help="Raise alarm if memory allocated is greater " "than given percentage") - form.addParam('memAlert', params.FloatParam, default=101, label="Raise Alarm if Memory > XX%", help="Raise alarm if cpu allocated is greater " @@ -135,9 +132,7 @@ def _defineParams(self, form): form.addSection('Mail settings') ProtMonitor._sendMailParams(self, form) - form.addSection('HTML Report') - form.addParam("doInflux", params.BooleanParam, label="use grafana/influx", default=False, @@ -146,7 +141,7 @@ def _defineParams(self, form): label="Publish command", help="Specify a command to publish the template. " "You can use the special token %(REPORT_FOLDER)s " - "that will be replaced with the report folder. " + "that will be replaced with the report folder." "For example: \n" "rsync -avL %(REPORT_FOLDER)s " "scipion@webserver:public_html/") @@ -190,12 +185,13 @@ def stepAll(): # when sysmonitor done all protocols done sysMonitorFinished = sysMonitor.step() htmlFinished = reportHtml.generate(finished) + if sysMonitorFinished and htmlFinished: finished = True reportHtml.generate(finished) except Exception as ex: - print("An error happened:") + print("An error happened:", ex) import traceback traceback.print_exc() @@ -217,6 +213,13 @@ def createReportDir(self): pathRepoSummary.write("HTML path to summary: " + self.reportPath) pathRepoSummary.close() + def _getPickingProtocol(self): + for protPointer in self.inputProtocols: + prot = protPointer.get() + print(type(prot)) + if isinstance(prot, ProtParticlePickingAuto): + return prot + return None def _getAlignProtocol(self): for protPointer in self.inputProtocols: @@ -305,8 +308,9 @@ def createSystemMonitor(self): doDiskIO=self.doDiskIO.get(), nif=MonitorSystem.getNifsNameList()[ self.netInterfaces.get()]) - + return sysMon + def getReportPath(self): return self.reportPath @@ -316,6 +320,7 @@ def createHtmlReport(self, ctfMonitor=None, sysMonitor=None, ctfMonitor = ctfMonitor or self.createCtfMonitor() sysMonitor = sysMonitor or self.createSystemMonitor() movieGainMonitor = movieGainMonitor or self.createMovieGainMonitor() + self.createReportDir() if self.doInflux: htmlReport = ReportInflux(self, ctfMonitor, sysMonitor, movieGainMonitor, @@ -328,6 +333,7 @@ def createHtmlReport(self, ctfMonitor=None, sysMonitor=None, htmlReport.setUp() return htmlReport + def _summary(self): summary = [] pathRepoSummary = self._getPath("pathRepo.txt") diff --git a/emfacilities/protocols/report_html.py b/emfacilities/protocols/report_html.py index d6e82434c..8498d12eb 100644 --- a/emfacilities/protocols/report_html.py +++ b/emfacilities/protocols/report_html.py @@ -33,6 +33,8 @@ import subprocess import multiprocessing from datetime import datetime +from PIL import Image as ImagePIL +from PIL import ImageDraw from statistics import median, mean from pyworkflow.protocol import getUpdatedProtocol @@ -48,6 +50,7 @@ MIC_PATH = 'imgMicPath' PSD_PATH = 'imgPsdPath' SHIFT_PATH = 'imgShiftPath' +PICK_PATH = 'imgPickPath' # These constants are the name of the folders where thumbnails # for the html report will be stored. They are also the keys to # used in the execution.summary.template.html to read data (where @@ -55,9 +58,11 @@ MIC_THUMBS = 'imgMicThumbs' PSD_THUMBS = 'imgPsdThumbs' SHIFT_THUMBS = 'imgShiftThumbs' +PICK_THUMBS = 'imgPickThumbs' MIC_ID = 'micId' DEFOCUS_HIST_BIN_WIDTH = 0.5 RESOLUTION_HIST_BIN_WIDTH = 0.5 +COORD = [] class ReportHtml: @@ -69,6 +74,7 @@ def __init__(self, protocol, ctfMonitor, sysMonitor, movieGainMonitor, publishCm self.protocol = protocol self.ctfProtocol = protocol._getCtfProtocol() self.alignProtocol = protocol._getAlignProtocol() + self.picking = protocol._getPickingProtocol() self.micThumbSymlinks = False self.reportPath = protocol.reportPath self.reportDir = protocol.reportDir @@ -91,11 +97,14 @@ def __init__(self, protocol, ctfMonitor, sysMonitor, movieGainMonitor, publishCm self.thumbPaths = {MIC_THUMBS: [], PSD_THUMBS: [], SHIFT_THUMBS: [], + PICK_THUMBS:[], MIC_PATH: [], SHIFT_PATH: [], PSD_PATH: [], - MIC_ID: []} - + PICK_PATH: [], + MIC_ID: [], + } + self.coordSet = [] # Get the html template to be used, by default use the one # in scipion/config/templates self.template = self._getHTMLTemplatePath() @@ -162,6 +171,24 @@ def setUp(self): and self.alignProtocol._doComputeMicThumbnail()): self.micThumbSymlinks = True + def getCoordset(self): + # TODO get this output names from Protocol constants + if hasattr(self.picking, 'outputCoordinates'): + return self.picking.outputCoordinates + elif hasattr(self.picking, 'outputCoordinates'): + return self.picking.outputCoordinates + else: + return None + + def getboxsice(self): + # TODO get this output names from Protocol constants + if hasattr(self.picking, 'boxsize'): + return self.picking.boxsize + elif hasattr(self.picking, 'boxsize'): + return self.picking.boxsize + else: + return None + def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='jpg', micIdSet=None): """Adds to self.thumbPaths the paths to the report thumbnails that come from the alignment and/or ctf protocol. @@ -213,6 +240,9 @@ def getMicSet(alignedProt): return else: return + + if self.picking is not None: + updatedProt = getUpdatedProtocol(self.picking) for micId in micIdSet[thumbsDone:]: mic = outputSet[micId] @@ -226,6 +256,11 @@ def getMicSet(alignedProt): self.thumbPaths[MIC_PATH].append(srcMicFn) self.thumbPaths[MIC_THUMBS].append(micThumbFn) + if self.picking is not None: + micThumbFn = join(PICK_THUMBS, pwutils.replaceExt(basename(srcMicFn), ext)) + self.thumbPaths[PICK_PATH].append(srcMicFn) + self.thumbPaths[PICK_THUMBS].append(micThumbFn) + shiftPlot = (getattr(mic, 'plotCart', None) or getattr(mic, 'plotGlobal', None)) if shiftPlot is not None: shiftPath = "" if shiftPlot is None else abspath(shiftPlot.getFileName()) @@ -262,6 +297,43 @@ def getMicPSDPath(mic): self.thumbPaths.pop(PSD_THUMBS, None) if PSD_PATH in self.thumbPaths: self.thumbPaths.pop(PSD_PATH, None) + + def plotParticlePicking(self): + """ + Function to plot 2D particle Picking + """ + coord = self.getCoordset() + boxsize = self.getboxsice() + mic = self.getCoordset().getMicrographs() + coordinatesDict = {} + i = 0 + numMics = len(self.thumbPaths[MIC_PATH]) + + for micrograph in mic: + if i <= numMics: + repPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) + coordinatesDict[micrograph.getMicName()] = {'path': repPath, 'Xdim': micrograph.getXDim(), + 'Ydim': micrograph.getYDim()} + i = i+1 + + for coordinate in coord: # for each micrograph, get its coordinates + if coordinate.getMicName() in coordinatesDict: + coordinatesDict[coordinate.getMicName()].setdefault('coords', []).append([coordinate.getX(), coordinate.getY()]) + + for micrograph, values in coordinatesDict.items(): # draw coordinates in micrographs jpgs + if 'coords' in values: + image = ImagePIL.open(values['path']).convert('RGB') + W_mic = values['Xdim'] + H_mic = values['Ydim'] + W_jpg, H_jpg = image.size + draw = ImageDraw.Draw(image) + r = int(boxsize)/2 + border_color = (0, 255, 0) # Set the border color here + for coord in values['coords']: + x = coord[0] * (W_jpg / W_mic) + y = coord[1] * (H_jpg / H_mic) + draw.ellipse((x - r, y - r, x + r, y + r), outline=border_color) + image.save(values['path'], quality=95) def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): """ Function to generate thumbnails for the report. Uses data from @@ -280,6 +352,7 @@ def generateReportImages(self, firstThumbIndex=0, micScaleFactor=6): print('Generating images for mic %d' % (i+1)) # mic thumbnails dstImgPath = join(self.reportDir, self.thumbPaths[MIC_THUMBS][i]) + if not exists(dstImgPath): if self.micThumbSymlinks: pwutils.copyFile(self.thumbPaths[MIC_PATH][i], dstImgPath) @@ -387,7 +460,6 @@ def getTimeSeries(self, data): # Get timeStamp ts = data[TIME_STAMP] - timeSeries = dict() # Get phaseShift @@ -528,6 +600,9 @@ def generate(self, finished): reportFinished = self.thumbsReady == numMics + if data: + self.plotParticlePicking() + def convert(o): if isinstance(o, np.int64): return int(o) raise TypeError