From 06b95ac5d57071519bd38bf809e2be11d26e482a Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Mon, 10 Apr 2023 15:38:30 +0100 Subject: [PATCH 01/16] Remove portalocker --- src/omero_ext/portalocker.py | 158 ----------------------------------- 1 file changed, 158 deletions(-) delete mode 100644 src/omero_ext/portalocker.py diff --git a/src/omero_ext/portalocker.py b/src/omero_ext/portalocker.py deleted file mode 100644 index 96d38fae3..000000000 --- a/src/omero_ext/portalocker.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# portalocker.py - Cross-platform (posix/nt) API for flock-style file locking. -# Requires python 1.5.2 or better. -"""Cross-platform (posix/nt) API for flock-style file locking. - -Synopsis: - - import portalocker - file = open("somefile", "r+") - portalocker.lock(file, portalocker.LOCK_EX) - file.seek(12) - file.write("foo") - file.close() - -If you know what you're doing, you may choose to - - portalocker.unlock(file) - -before closing the file, but why? - -Methods: - - lock( file, flags ) - unlock( file ) - -Constants: - - LOCK_EX - LOCK_SH - LOCK_NB - -Exceptions: - - LockException - -Notes: - -For the 'nt' platform, this module requires the Python Extensions for Windows. -Be aware that this may not work as expected on Windows 95/98/ME. - -History: - -I learned the win32 technique for locking files from sample code -provided by John Nielsen in the documentation -that accompanies the win32 modules. - -Author: Jonathan Feinberg , - Lowell Alleman -Version: $Id: portalocker.py 5474 2008-05-16 20:53:50Z lowell $ - -""" -from __future__ import print_function -from __future__ import absolute_import - - -__all__ = [ - "lock", - "unlock", - "LOCK_EX", - "LOCK_SH", - "LOCK_NB", - "LockException", -] - -import os - -class LockException(Exception): - # Error codes: - LOCK_FAILED = 1 - -if os.name == 'nt': - import win32con - import win32file - import pywintypes - LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK - LOCK_SH = 0 # the default - LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY - # is there any reason not to reuse the following structure? - __overlapped = pywintypes.OVERLAPPED() -elif os.name == 'posix': - import fcntl - LOCK_EX = fcntl.LOCK_EX - LOCK_SH = fcntl.LOCK_SH - LOCK_NB = fcntl.LOCK_NB -else: - raise RuntimeError("PortaLocker only defined for nt and posix platforms") - -if os.name == 'nt': - - def lockno(fileno, flags): - hfile = win32file._get_osfhandle(fileno) - try: - win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped) - except pywintypes.error as exc_value: - # error: (33, 'LockFileEx', 'The process cannot access the file because another process has locked a portion of the file.') - if exc_value.args[0] == 33: - raise LockException( - LockException.LOCK_FAILED, exc_value.args[2]) - else: - # Q: Are there exceptions/codes we should be dealing with here? - raise - - def unlockno(fileno): - hfile = win32file._get_osfhandle(fileno) - try: - win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped) - except pywintypes.error as exc_value: - if exc_value[0] == 158: - # error: (158, 'UnlockFileEx', 'The segment is already unlocked.') - # To match the 'posix' implementation, silently ignore this error - pass - else: - # Q: Are there exceptions/codes we should be dealing with here? - raise - -elif os.name == 'posix': - def lockno(fileno, flags): - try: - fcntl.flock(fileno, flags) - except IOError as exc_value: - # IOError: [Errno 11] Resource temporarily unavailable - # Following added by Glencoe Software, Inc. using LOCK_NB|LOCK_EX on Mac 10.4 - # IOError: [Errno 35] Resource temporarily unavailable - if exc_value.args[0] == 11 or exc_value.args[0] == 35: - raise LockException( - LockException.LOCK_FAILED, exc_value.args[1]) - else: - raise - - def unlockno(fileno): - fcntl.flock(fileno, fcntl.LOCK_UN) - - -def lock(file, flags): - lockno(file.fileno(), flags) - - -def unlock(file): - unlockno(file.fileno()) - - -if __name__ == '__main__': - from time import time, strftime, localtime - import sys - from . import portalocker - - log = open('log.txt', "a+") - portalocker.lock(log, portalocker.LOCK_EX) - - timestamp = strftime("%m/%d/%Y %H:%M:%S\n", localtime(time())) - log.write( timestamp ) - - print("Wrote lines. Hit enter to release lock.") - dummy = sys.stdin.readline() - - log.close() - From 2d4ded2ce58977cfbb3d7dd90c90a417f6a056da Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Mon, 10 Apr 2023 15:38:41 +0100 Subject: [PATCH 02/16] install portalocker --- setup.py | 3 ++- src/omero/config.py | 2 +- src/omero/hdfstorageV2.py | 2 +- src/omero/plugins/admin.py | 2 +- src/omero/plugins/prefs.py | 2 +- src/omero/util/temp_files.py | 2 +- src/omero_ext/cloghandler.py | 2 +- test/unit/test_config.py | 2 +- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 317135f54..e8c1fcd37 100644 --- a/setup.py +++ b/setup.py @@ -227,7 +227,8 @@ def read(fname): 'PyYAML', 'zeroc-ice>=3.6.5,<3.7', 'pywin32; platform_system=="Windows"', - 'requests' + 'requests', + 'portalocker' ], tests_require=[ 'pytest', diff --git a/src/omero/config.py b/src/omero/config.py index 558da6d74..c72f3b45c 100644 --- a/src/omero/config.py +++ b/src/omero/config.py @@ -30,7 +30,7 @@ XML, Element, ElementTree, SubElement, Comment, tostring ) import omero_ext.path as path -from omero_ext import portalocker +import portalocker import json diff --git a/src/omero/hdfstorageV2.py b/src/omero/hdfstorageV2.py index ee499d516..ed1ada383 100644 --- a/src/omero/hdfstorageV2.py +++ b/src/omero/hdfstorageV2.py @@ -32,7 +32,7 @@ from omero.rtypes import rfloat, rlong, rstring, unwrap from omero.util.decorators import locked from omero_ext.path import path -from omero_ext import portalocker +import portalocker from functools import wraps diff --git a/src/omero/plugins/admin.py b/src/omero/plugins/admin.py index 31b97ae97..a0e0546ec 100755 --- a/src/omero/plugins/admin.py +++ b/src/omero/plugins/admin.py @@ -50,7 +50,7 @@ WriteableConfigControl, with_config from omero.install.windows_warning import windows_warning, WINDOWS_WARNING -from omero_ext import portalocker +import portalocker from omero_ext.path import path from omero_ext.which import whichall from omero_ext.argparse import FileType diff --git a/src/omero/plugins/prefs.py b/src/omero/plugins/prefs.py index 5a28f832b..d19acd5e6 100755 --- a/src/omero/plugins/prefs.py +++ b/src/omero/plugins/prefs.py @@ -28,7 +28,7 @@ from omero.util import edit_path, get_omero_userdir from omero.util.decorators import wraps from omero.util.upgrade_check import UpgradeCheck -from omero_ext import portalocker +import portalocker from omero_ext.argparse import SUPPRESS from omero_ext.path import path diff --git a/src/omero/util/temp_files.py b/src/omero/util/temp_files.py index c39c6144a..d70c9d232 100644 --- a/src/omero/util/temp_files.py +++ b/src/omero/util/temp_files.py @@ -21,7 +21,7 @@ import tempfile from omero.util import get_omero_userdir, get_user -from omero_ext import portalocker +import portalocker from omero_ext.path import path # Activating logging at a static level diff --git a/src/omero_ext/cloghandler.py b/src/omero_ext/cloghandler.py index 9e8ebf656..eea8e2edb 100644 --- a/src/omero_ext/cloghandler.py +++ b/src/omero_ext/cloghandler.py @@ -75,7 +75,7 @@ # clobbering that the builtin class allows. # sibling module than handles all the ugly platform-specific details of file locking -from .portalocker import lock, unlock, LOCK_EX, LOCK_NB, LockException +from portalocker import lock, unlock, LOCK_EX, LOCK_NB, LockException # Workaround for handleError() in Python 2.7+ where record is written to stderr diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 8d8b24203..4f09a92a9 100755 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -19,7 +19,7 @@ import errno import pytest from omero.config import ConfigXml, xml -from omero_ext import portalocker +import portalocker from xml.etree.ElementTree import XML, Element, SubElement, tostring From e477e830232e160eeffcf125916be3479450dd4c Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Tue, 18 Apr 2023 08:38:02 +0100 Subject: [PATCH 03/16] fix method --- src/omero/hdfstorageV2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/omero/hdfstorageV2.py b/src/omero/hdfstorageV2.py index ed1ada383..189ed2f0b 100644 --- a/src/omero/hdfstorageV2.py +++ b/src/omero/hdfstorageV2.py @@ -137,7 +137,7 @@ def addOrThrow(self, hdfpath, hdfstorage, read_only=False): if not read_only: try: - portalocker.lockno( + portalocker.lock( fileno, portalocker.LOCK_NB | portalocker.LOCK_EX) except portalocker.LockException: hdffile.close() From 20af12ac5fd581190bd44ba5be3f1019780aaa75 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Tue, 18 Apr 2023 10:36:42 +0100 Subject: [PATCH 04/16] review existence of file --- src/omero/util/temp_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omero/util/temp_files.py b/src/omero/util/temp_files.py index d70c9d232..f8d5226f9 100644 --- a/src/omero/util/temp_files.py +++ b/src/omero/util/temp_files.py @@ -241,8 +241,8 @@ def create(self, dir): If the given directory doesn't exist, creates it (with mode 0700) and returns True. Otherwise False. """ - dir = path(dir) - if not dir.exists(): + # dir = path(dir) + if not os.path.exists(dir): dir.makedirs(0o700) return True return False From c88358cf423733c3ac05f2839fec5f6e8963f488 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Wed, 19 Apr 2023 12:29:24 +0100 Subject: [PATCH 05/16] fix method --- src/omero/hdfstorageV2.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/omero/hdfstorageV2.py b/src/omero/hdfstorageV2.py index 189ed2f0b..ee273606b 100644 --- a/src/omero/hdfstorageV2.py +++ b/src/omero/hdfstorageV2.py @@ -21,6 +21,7 @@ import threading import traceback +import os from os import W_OK import omero # Do we need both?? @@ -134,13 +135,11 @@ def addOrThrow(self, hdfpath, hdfstorage, read_only=False): mode = read_only and "r" or "a" hdffile = hdfstorage.openfile(mode) fileno = hdffile.fileno() - if not read_only: try: - portalocker.lock( - fileno, portalocker.LOCK_NB | portalocker.LOCK_EX) + portalocker.Lock( + hdfpath, flags=(portalocker.LOCK_NB | portalocker.LOCK_EX)) except portalocker.LockException: - hdffile.close() raise omero.LockTimeout( None, None, "Cannot acquire exclusive lock on: %s" % hdfpath, 0) From 1d457283502d19734132747dd86de13f3205a6d5 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Fri, 21 Apr 2023 09:27:32 +0100 Subject: [PATCH 06/16] remove import introduced during testing --- src/omero/hdfstorageV2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/omero/hdfstorageV2.py b/src/omero/hdfstorageV2.py index ee273606b..040975c95 100644 --- a/src/omero/hdfstorageV2.py +++ b/src/omero/hdfstorageV2.py @@ -21,7 +21,6 @@ import threading import traceback -import os from os import W_OK import omero # Do we need both?? From 326d43c5ab0b76a4a6bd2d42603cf4e2ddbcb0f3 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Mon, 24 Apr 2023 10:08:01 +0100 Subject: [PATCH 07/16] remove cloghandler --- src/omero_ext/cloghandler.py | 351 ----------------------------------- 1 file changed, 351 deletions(-) delete mode 100644 src/omero_ext/cloghandler.py diff --git a/src/omero_ext/cloghandler.py b/src/omero_ext/cloghandler.py deleted file mode 100644 index eea8e2edb..000000000 --- a/src/omero_ext/cloghandler.py +++ /dev/null @@ -1,351 +0,0 @@ -# Copyright 2013 Lowell Alleman -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" cloghandler.py: A smart replacement for the standard RotatingFileHandler - -ConcurrentRotatingFileHandler: This class is a log handler which is a drop-in -replacement for the python standard log handler 'RotateFileHandler', the primary -difference being that this handler will continue to write to the same file if -the file cannot be rotated for some reason, whereas the RotatingFileHandler will -strictly adhere to the maximum file size. Unfortunately, if you are using the -RotatingFileHandler on Windows, you will find that once an attempted rotation -fails, all subsequent log messages are dropped. The other major advantage of -this module is that multiple processes can safely write to a single log file. - -To put it another way: This module's top priority is preserving your log -records, whereas the standard library attempts to limit disk usage, which can -potentially drop log messages. If you are trying to determine which module to -use, there are number of considerations: What is most important: strict disk -space usage or preservation of log messages? What OSes are you supporting? Can -you afford to have processes blocked by file locks? - -Concurrent access is handled by using file locks, which should ensure that log -messages are not dropped or clobbered. This means that a file lock is acquired -and released for every log message that is written to disk. (On Windows, you may -also run into a temporary situation where the log file must be opened and closed -for each log message.) This can have potentially performance implications. In my -testing, performance was more than adequate, but if you need a high-volume or -low-latency solution, I suggest you look elsewhere. - -This module currently only support the 'nt' and 'posix' platforms due to the -usage of the portalocker module. I do not have access to any other platforms -for testing, patches are welcome. - -See the README file for an example usage of this module. - -This module supports Python 2.6 and later. - -""" -from __future__ import absolute_import - - -from builtins import range -__version__ = '0.9.1' -__revision__ = 'lowell87@gmail.com-20130711022321-doutxl7zyzuwss5a 2013-07-10 22:23:21 -0400 [0]' -__author__ = "Lowell Alleman" -__all__ = [ - "ConcurrentRotatingFileHandler", -] - - -import os -import sys -from random import randint -from logging import Handler, LogRecord -from logging.handlers import BaseRotatingHandler - -try: - import codecs -except ImportError: - codecs = None - - - -# Question/TODO: Should we have a fallback mode if we can't load portalocker / -# we should still be better off than with the standard RotattingFileHandler -# class, right? We do some rename checking... that should prevent some file -# clobbering that the builtin class allows. - -# sibling module than handles all the ugly platform-specific details of file locking -from portalocker import lock, unlock, LOCK_EX, LOCK_NB, LockException - - -# Workaround for handleError() in Python 2.7+ where record is written to stderr -class NullLogRecord(LogRecord): - def __init__(self): - pass - def __getattr__(self, attr): - return None - -class ConcurrentRotatingFileHandler(BaseRotatingHandler): - """ - Handler for logging to a set of files, which switches from one file to the - next when the current file reaches a certain size. Multiple processes can - write to the log file concurrently, but this may mean that the file will - exceed the given size. - """ - def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, - encoding=None, debug=True, delay=0): - """ - Open the specified file and use it as the stream for logging. - - By default, the file grows indefinitely. You can specify particular - values of maxBytes and backupCount to allow the file to rollover at - a predetermined size. - - Rollover occurs whenever the current log file is nearly maxBytes in - length. If backupCount is >= 1, the system will successively create - new files with the same pathname as the base file, but with extensions - ".1", ".2" etc. appended to it. For example, with a backupCount of 5 - and a base file name of "app.log", you would get "app.log", - "app.log.1", "app.log.2", ... through to "app.log.5". The file being - written to is always "app.log" - when it gets filled up, it is closed - and renamed to "app.log.1", and if files "app.log.1", "app.log.2" etc. - exist, then they are renamed to "app.log.2", "app.log.3" etc. - respectively. - - If maxBytes is zero, rollover never occurs. - - On Windows, it is not possible to rename a file that is currently opened - by another process. This means that it is not possible to rotate the - log files if multiple processes is using the same log file. In this - case, the current log file will continue to grow until the rotation can - be completed successfully. In order for rotation to be possible, all of - the other processes need to close the file first. A mechanism, called - "degraded" mode, has been created for this scenario. In degraded mode, - the log file is closed after each log message is written. So once all - processes have entered degraded mode, the net rotation attempt should - be successful and then normal logging can be resumed. Using the 'delay' - parameter may help reduce contention in some usage patterns. - - This log handler assumes that all concurrent processes logging to a - single file will are using only this class, and that the exact same - parameters are provided to each instance of this class. If, for - example, two different processes are using this class, but with - different values for 'maxBytes' or 'backupCount', then odd behavior is - expected. The same is true if this class is used by one application, but - the RotatingFileHandler is used by another. - """ - # Absolute file name handling done by FileHandler since Python 2.5 - BaseRotatingHandler.__init__(self, filename, mode, encoding, delay) - self.delay = delay - self._rotateFailed = False - self.maxBytes = maxBytes - self.backupCount = backupCount - self._open_lockfile() - # For debug mode, swap out the "_degrade()" method with a more a verbose one. - if debug: - self._degrade = self._degrade_debug - - def _open_lockfile(self): - # Use 'file.lock' and not 'file.log.lock' (Only handles the normal "*.log" case.) - if self.baseFilename.endswith(".log"): - lock_file = self.baseFilename[:-4] - else: - lock_file = self.baseFilename - lock_file += ".lock" - self.stream_lock = open(lock_file,"w") - - def _open(self, mode=None): - """ - Open the current base file with the (original) mode and encoding. - Return the resulting stream. - - Note: Copied from stdlib. Added option to override 'mode' - """ - if mode is None: - mode = self.mode - if self.encoding is None: - stream = open(self.baseFilename, mode) - else: - stream = codecs.open(self.baseFilename, mode, self.encoding) - return stream - - def _close(self): - """ Close file stream. Unlike close(), we don't tear anything down, we - expect the log to be re-opened after rotation.""" - if self.stream: - try: - if not self.stream.closed: - # Flushing probably isn't technically necessary, but it feels right - self.stream.flush() - self.stream.close() - finally: - self.stream = None - - def acquire(self): - """ Acquire thread and file locks. Re-opening log for 'degraded' mode. - """ - # handle thread lock - Handler.acquire(self) - # Issue a file lock. (This is inefficient for multiple active threads - # within a single process. But if you're worried about high-performance, - # you probably aren't using this log handler.) - if self.stream_lock: - # If stream_lock=None, then assume close() was called or something - # else weird and ignore all file-level locks. - if self.stream_lock.closed: - # Daemonization can close all open file descriptors, see - # https://bugzilla.redhat.com/show_bug.cgi?id=952929 - # Try opening the lock file again. Should we warn() here?!? - try: - self._open_lockfile() - except Exception: - self.handleError(NullLogRecord()) - # Don't try to open the stream lock again - self.stream_lock = None - return - lock(self.stream_lock, LOCK_EX) - # Stream will be opened as part by FileHandler.emit() - - def release(self): - """ Release file and thread locks. If in 'degraded' mode, close the - stream to reduce contention until the log files can be rotated. """ - try: - if self._rotateFailed: - self._close() - except Exception: - self.handleError(NullLogRecord()) - finally: - try: - if self.stream_lock and not self.stream_lock.closed: - unlock(self.stream_lock) - except Exception: - self.handleError(NullLogRecord()) - finally: - # release thread lock - Handler.release(self) - - def close(self): - """ - Close log stream and stream_lock. """ - try: - self._close() - if not self.stream_lock.closed: - self.stream_lock.close() - finally: - self.stream_lock = None - Handler.close(self) - - def _degrade(self, degrade, msg, *args): - """ Set degrade mode or not. Ignore msg. """ - self._rotateFailed = degrade - del msg, args # avoid pychecker warnings - - def _degrade_debug(self, degrade, msg, *args): - """ A more colorful version of _degade(). (This is enabled by passing - "debug=True" at initialization). - """ - if degrade: - if not self._rotateFailed: - sys.stderr.write("Degrade mode - ENTERING - (pid=%d) %s\n" % - (os.getpid(), msg % args)) - self._rotateFailed = True - else: - if self._rotateFailed: - sys.stderr.write("Degrade mode - EXITING - (pid=%d) %s\n" % - (os.getpid(), msg % args)) - self._rotateFailed = False - - def doRollover(self): - """ - Do a rollover, as described in __init__(). - """ - self._close() - if self.backupCount <= 0: - # Don't keep any backups, just overwrite the existing backup file - # Locking doesn't much matter here; since we are overwriting it anyway - self.stream = self._open("w") - return - try: - # Determine if we can rename the log file or not. Windows refuses to - # rename an open file, Unix is inode base so it doesn't care. - - # Attempt to rename logfile to tempname: There is a slight race-condition here, but it seems unavoidable - tmpname = None - while not tmpname or os.path.exists(tmpname): - tmpname = "%s.rotate.%08d" % (self.baseFilename, randint(0,99999999)) - try: - # Do a rename test to determine if we can successfully rename the log file - os.rename(self.baseFilename, tmpname) - except (IOError, OSError): - exc_value = sys.exc_info()[1] - self._degrade(True, "rename failed. File in use? " - "exception=%s", exc_value) - return - - # Q: Is there some way to protect this code from a KeboardInterupt? - # This isn't necessarily a data loss issue, but it certainly does - # break the rotation process during stress testing. - - # There is currently no mechanism in place to handle the situation - # where one of these log files cannot be renamed. (Example, user - # opens "logfile.3" in notepad); we could test rename each file, but - # nobody's complained about this being an issue; so the additional - # code complexity isn't warranted. - for i in range(self.backupCount - 1, 0, -1): - sfn = "%s.%d" % (self.baseFilename, i) - dfn = "%s.%d" % (self.baseFilename, i + 1) - if os.path.exists(sfn): - #print "%s -> %s" % (sfn, dfn) - if os.path.exists(dfn): - os.remove(dfn) - os.rename(sfn, dfn) - dfn = self.baseFilename + ".1" - if os.path.exists(dfn): - os.remove(dfn) - os.rename(tmpname, dfn) - #print "%s -> %s" % (self.baseFilename, dfn) - self._degrade(False, "Rotation completed") - finally: - # Re-open the output stream, but if "delay" is enabled then wait - # until the next emit() call. This could reduce rename contention in - # some usage patterns. - if not self.delay: - self.stream = self._open() - - def shouldRollover(self, record): - """ - Determine if rollover should occur. - - For those that are keeping track. This differs from the standard - library's RotatingLogHandler class. Because there is no promise to keep - the file size under maxBytes we ignore the length of the current record. - """ - del record # avoid pychecker warnings - # Is stream is not yet open, skip rollover check. (Check will occur on - # next message, after emit() calls _open()) - if self.stream is None: - return False - if self._shouldRollover(): - # If some other process already did the rollover (which is possible - # on Unix) the file our stream may now be named "log.1", thus - # triggering another rollover. Avoid this by closing and opening - # "log" again. - self._close() - self.stream = self._open() - return self._shouldRollover() - return False - - def _shouldRollover(self): - if self.maxBytes > 0: # are we rolling over? - self.stream.seek(0, 2) #due to non-posix-compliant Windows feature - if self.stream.tell() >= self.maxBytes: - return True - else: - self._degrade(False, "Rotation done or not needed at this time") - return False - - -# Publish this class to the "logging.handlers" module so that it can be use -# from a logging config file via logging.config.fileConfig(). -import logging.handlers -logging.handlers.ConcurrentRotatingFileHandler = ConcurrentRotatingFileHandler From b0a8c3c31527cdf1b2790a4d6edd57bd231ef5fe Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Mon, 24 Apr 2023 10:09:10 +0100 Subject: [PATCH 08/16] re-introduce line removed by error --- src/omero/hdfstorageV2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/omero/hdfstorageV2.py b/src/omero/hdfstorageV2.py index 040975c95..80e08c840 100644 --- a/src/omero/hdfstorageV2.py +++ b/src/omero/hdfstorageV2.py @@ -139,6 +139,7 @@ def addOrThrow(self, hdfpath, hdfstorage, read_only=False): portalocker.Lock( hdfpath, flags=(portalocker.LOCK_NB | portalocker.LOCK_EX)) except portalocker.LockException: + hdffile.close() raise omero.LockTimeout( None, None, "Cannot acquire exclusive lock on: %s" % hdfpath, 0) From 210325ad5cf42ec87b9eb85631d183b8e3bfb255 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Tue, 25 Apr 2023 16:44:35 +0100 Subject: [PATCH 09/16] use lock method --- src/omero/hdfstorageV2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omero/hdfstorageV2.py b/src/omero/hdfstorageV2.py index 80e08c840..121994844 100644 --- a/src/omero/hdfstorageV2.py +++ b/src/omero/hdfstorageV2.py @@ -136,8 +136,8 @@ def addOrThrow(self, hdfpath, hdfstorage, read_only=False): fileno = hdffile.fileno() if not read_only: try: - portalocker.Lock( - hdfpath, flags=(portalocker.LOCK_NB | portalocker.LOCK_EX)) + portalocker.lock( + hdffile, flags=(portalocker.LOCK_NB | portalocker.LOCK_EX)) except portalocker.LockException: hdffile.close() raise omero.LockTimeout( From 0d0d0cc0a5657e141d12feee3414cfe067120df1 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Tue, 25 Apr 2023 16:44:59 +0100 Subject: [PATCH 10/16] drop windows --- .github/workflows/workflow.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 374553b24..ab1b8abfb 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -22,8 +22,6 @@ jobs: include: - python-version: '3.8' os: macos-latest - - python-version: '3.7' - os: windows-latest runs-on: ${{ matrix.os }} steps: - name: Get tox target From 0c4f3f2b20574cdb3783207747b144a3079d6643 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Wed, 3 May 2023 11:34:33 +0100 Subject: [PATCH 11/16] run build on windows with Python 3.8 --- .github/workflows/workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index ab1b8abfb..657ba379b 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -22,6 +22,8 @@ jobs: include: - python-version: '3.8' os: macos-latest + - python-version: '3.8' + os: windows-latest runs-on: ${{ matrix.os }} steps: - name: Get tox target From 368a9d386f0e283daf015da0616002a29ff688c0 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Wed, 3 May 2023 11:34:49 +0100 Subject: [PATCH 12/16] skip test if windows --- test/unit/tablestest/test_hdfstorage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/tablestest/test_hdfstorage.py b/test/unit/tablestest/test_hdfstorage.py index a7e254641..43440bf24 100755 --- a/test/unit/tablestest/test_hdfstorage.py +++ b/test/unit/tablestest/test_hdfstorage.py @@ -29,7 +29,10 @@ from omero_ext.path import path import omero.hdfstorageV2 as storage_module +import sys +if sys.platform.startswith("win"): + pytest.skip("skipping tests on windows", allow_module_level=True) HdfList = storage_module.HdfList HdfStorage = storage_module.HdfStorage From 66cd6f4248ea5dbc65a21677b8412298b472b567 Mon Sep 17 00:00:00 2001 From: Jean-Marie Burel Date: Wed, 3 May 2023 11:50:13 +0100 Subject: [PATCH 13/16] do not run test on windows --- test/unit/tablestest/test_servants.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/unit/tablestest/test_servants.py b/test/unit/tablestest/test_servants.py index 5737d4592..b5419c043 100755 --- a/test/unit/tablestest/test_servants.py +++ b/test/unit/tablestest/test_servants.py @@ -26,6 +26,10 @@ from omero.columns import LongColumnI, DoubleColumnI, ObjectFactories from omero_ext.path import path +import sys +if sys.platform.startswith("win"): + pytest.skip("skipping tests on windows", allow_module_level=True) + logging.basicConfig(level=logging.DEBUG) # Don't retry since we expect errors From cbf7bd5e50d2931c775bdbbc3515f5390d3a1b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Besson?= Date: Fri, 25 Aug 2023 16:08:09 +0100 Subject: [PATCH 14/16] Re-activate OMERO.tables testGetSetMetaMap As of the migration to Python 3 and f8af68d771f34d8b2ba16fc28c58aa7d85fb70e3, both int and long values are returned as rlong --- test/unit/tablestest/test_hdfstorage.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/unit/tablestest/test_hdfstorage.py b/test/unit/tablestest/test_hdfstorage.py index 43440bf24..f6f6281b1 100755 --- a/test/unit/tablestest/test_hdfstorage.py +++ b/test/unit/tablestest/test_hdfstorage.py @@ -23,7 +23,7 @@ import Ice from mox3 import mox -from omero.rtypes import rint, rstring +from omero.rtypes import rint, rstring, rlong from library import TestCase from omero_ext.path import path @@ -228,8 +228,6 @@ def testHandlesExistingDirectory(self): hdf = HdfStorage(h, self.lock) hdf.cleanup() - @pytest.mark.xfail - @pytest.mark.broken(reason = "TODO after python3 migration") def testGetSetMetaMap(self): hdf = HdfStorage(self.hdfpath(), self.lock) self.init(hdf, False) @@ -239,7 +237,7 @@ def testGetSetMetaMap(self): assert len(m1) == 3 assert m1['__initialized'].val > 0 assert m1['__version'] == rstring('2') - assert m1['a'] == rint(1) + assert m1['a'] == rlong(1) with pytest.raises(omero.ApiUsageException) as exc: hdf.add_meta_map({'b': rint(1), '__c': rint(2)}) @@ -259,7 +257,7 @@ def testGetSetMetaMap(self): hdf.add_meta_map({'__test': 1}, replace=True, init=True) m3 = hdf.get_meta_map() - assert m3 == {'__test': rint(1)} + assert m3 == {'__test': rlong(1)} hdf.cleanup() From 7e24e70a16ae0f1b02a36f7fd7a737baf6e6e077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Besson?= Date: Fri, 25 Aug 2023 16:09:58 +0100 Subject: [PATCH 15/16] Re-activate OMERO.tables TestHdfList.testLocking - Expect tables.open_file to open the file in append mode - Remove unnecessary print and FILE_OPEN_POLICY statement - Remove conditional handling of Pytables 2.x openFile API - Remove Pytables ValueError check (possibly invalid since Pytable 3.1) --- test/unit/tablestest/test_hdfstorage.py | 32 ++++--------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/test/unit/tablestest/test_hdfstorage.py b/test/unit/tablestest/test_hdfstorage.py index f6f6281b1..3fa42e230 100755 --- a/test/unit/tablestest/test_hdfstorage.py +++ b/test/unit/tablestest/test_hdfstorage.py @@ -500,8 +500,6 @@ def hdfpath(self): tmpdir = self.tmpdir() return old_div(path(tmpdir), "test.h5") - @pytest.mark.xfail - @pytest.mark.broken(reason = "TODO after python3 migration") def testLocking(self, monkeypatch): lock1 = threading.RLock() hdflist2 = HdfList() @@ -511,36 +509,16 @@ def testLocking(self, monkeypatch): # Using HDFLIST hdf1 = HdfStorage(tmp, lock1) - # There are multiple guards against opening the same HDF5 file - - # PyTables includes a check - monkeypatch.setattr(storage_module, 'HDFLIST', hdflist2) - with pytest.raises(ValueError) as exc_info: - HdfStorage(tmp, lock2) - - assert exc_info.value.message.startswith( - "The file '%s' is already opened. " % tmp) - monkeypatch.undo() - # HdfList uses portalocker, test by mocking tables.open_file - if hasattr(tables, "open_file"): - self.mox.StubOutWithMock(tables, 'open_file') - tables.file._FILE_OPEN_POLICY = 'default' - tables.open_file(tmp, mode='w', - title='OMERO HDF Measurement Storage', - rootUEP='/').AndReturn(open(tmp)) - - self.mox.ReplayAll() - else: - self.mox.StubOutWithMock(tables, 'openFile') - tables.openFile(tmp, mode='w', - title='OMERO HDF Measurement Storage', - rootUEP='/').AndReturn(open(tmp)) + self.mox.StubOutWithMock(tables, 'open_file') + tables.open_file(tmp, mode='a', + title='OMERO HDF Measurement Storage', + rootUEP='/').AndReturn(open(tmp)) + self.mox.ReplayAll() monkeypatch.setattr(storage_module, 'HDFLIST', hdflist2) with pytest.raises(omero.LockTimeout) as exc_info: HdfStorage(tmp, lock2) - print(exc_info.value) assert (exc_info.value.message == 'Cannot acquire exclusive lock on: %s' % tmp) From f11d3628fd97f882e1b2df3c3f7e43ad6938cbf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Besson?= Date: Wed, 30 Aug 2023 11:19:05 +0100 Subject: [PATCH 16/16] Use verbose mode to list all executed tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 46ad4e763..c58718a7a 100644 --- a/tox.ini +++ b/tox.ini @@ -35,5 +35,5 @@ passenv = commands = rst-lint README.rst python setup.py install - pytest {posargs:-n4 -m "not broken" --reruns 5 -rf test -s} + pytest {posargs:-n4 -m "not broken" --reruns 5 -rf test -sv} omero version