Service file for the filesystem monitor

app/src/onedrive_client/utils/constants.py

@@ -0,0 +1,18 @@
+"""
+Common constants
+"""
+
+ACTIONS = {
+    'start': 'start the service',
+    'restart': 'stop and restart the service if the service '
+               'is already running, otherwise start the service',
+    'stop': 'stop the service',
+    'try-restart': 'restart the service if the service '
+                   'is already running',
+    'reload': 'cause the configuration of the service to be reloaded '
+              'without actually stopping and restarting the service',
+    'force-reload': 'cause the configuration to be reloaded if the '
+                    'service supports this, otherwise restart the '
+                    'service if it is running',
+    'status': 'print the current status of the service'
+}

app/src/onedrive_client/utils/daemon.py

@@ -49,9 +49,9 @@ def stop(self, force=True):
 
         pid = self.get_pid()
         if not pid:
-            message = 'pidfile %s does not exist. ' + \
-                      'Daemon not running?\n'
-            print(message, self.pid)
+            message = 'pidfile %s does not exist. '\
+                      'Daemon not running?\n' % str(self.pid)
+            print(message)
             # not an error in a restart
             if force:
                 return 0

entities/proto/onedrive_client/entities/common.proto

@@ -16,6 +16,8 @@ import "onedrive_client/entities/onedrive.proto";
 message LocalItemMetadata {
   uint64 size = 1;
   uint64 modified = 2;
+  bool is_dir = 3;
+  bool is_deleted = 4;
 }
 
 

entities/python/src/onedrive_client/entities/base.py

@@ -165,11 +165,14 @@ class Entity(collections.abc.MutableMapping, metaclass=_EntityMeta):
 
     Complex fields are automatically converted to instances of
     the corresponding entity-classes:
-    >>> local_item['metadata'] = {'size': 999, 'modified': 123456789}
+    >>> local_item['metadata'] = {'size': 999, 'modified': 123456789,\
+                                  'is_dir': True}
     >>> local_item
     LocalItem({'path': '/bar',
                'metadata': LocalItemMetadata({'size': 999,
-                                              'modified': 123456789})})
+                                              'modified': 123456789,
+                                              'is_dir': True,
+                                              'is_deleted': False})})
 
     Repeated fields. Notice repeated fields are set by default unlike
     complex fields. It's because it's Protobuf specs:

filesystem_service/src/filesystem_service/fsmonitor

@@ -0,0 +1,93 @@
+#!/usr/bin/python
+
+"""
+Service module for the filesystem monitoring
+"""
+
+import argparse
+from pathlib import Path
+
+from filesystem_service.monitor import FileSystemMonitor
+from onedrive_client.utils.constants import ACTIONS
+from onedrive_client.utils.daemon import Daemon
+
+_FILESYSTEM_MONITOR = 'fsmonitor'
+_FILESYSTEM_MONITOR_VERSION = '0.1.0'
+_CONFIG_PATH = '/etc/fsmonitor.yml'
+_PID_FILE = '/var/run/fsmonitor.pid'
+
+
+# pylint: disable=bare-except
+# pylint: disable=too-few-public-methods
+class Monitor(object):
+    """
+    Monitoring starter
+    """
+
+    def __init__(self):
+        try:
+            self.config = load_config()
+        # ToDo: add proper exception
+        except:
+            print('Failed to load config')
+            exit(1)
+
+    def start(self):
+        """
+        Starts monitoring of file system events
+        """
+
+        mon = FileSystemMonitor(self.config.get('watch', str(Path.home())))
+        # ToDo: add subscribers
+        for exclude in self.config.get('exclude', []):
+            mon.add_exclude_folder(exclude)
+        mon.monitor()
+
+
+def load_config():
+    """
+    :return:
+    """
+
+    return dict()
+
+
+def main():
+    """
+    :return:
+    """
+
+    desc = 'Starts monitoring of specified local folder'
+    epilog = 'OneDrive-L Filesystem Monitoring Service'
+
+    parser = argparse.ArgumentParser(prog='filesystem-monitor',
+                                     description=desc, epilog=epilog)
+    parser.add_argument('-v', '--version', dest='version', action='version',
+                        version='%(prog)s ' + _FILESYSTEM_MONITOR_VERSION,
+                        help='Display version information')
+
+    subparsers = parser.add_subparsers(title='Commands')
+    for action, help_msg in ACTIONS.items():
+        setup = subparsers.add_parser(action, help=help_msg)
+        setup.set_defaults(which=action)
+
+    args = parser.parse_args()
+
+    fsmonitor = Monitor()
+    daemon = Daemon(app="filesystem-monitor", pid=_PID_FILE,
+                    action=fsmonitor.start)
+
+    if args.which == 'start':
+        return daemon.start()
+    elif args.which == 'stop':
+        return daemon.stop()
+    elif args.which in ('restart', 'reload'):
+        return daemon.restart()
+    elif args.which in ('try-restart', 'force-reload'):
+        return daemon.try_restart()
+    elif args.which == 'status':
+        return daemon.try_restart()
+
+
+if __name__ == '__main__':
+    exit(main())

filesystem_service/src/filesystem_service/monitor.py

@@ -12,6 +12,9 @@ class FileSystemMonitor(object):
     Monitor local filesystem changes and inform subscribers
     """
 
+    important_events = ('IN_CLOSE_WRITE', 'IN_CREATE', 'IN_DELETE',
+                        'IN_MOVED_FROM', 'IN_MOVED_TO', 'IN_ATTRIB')
+
     def __init__(self, folder):
         """
         :param folder: folder to be watched. Type: string
@@ -21,6 +24,7 @@ def __init__(self, folder):
         self._exclude_folders = set()
         self._subscribers = set()
         self._notifier = inotify.adapters.Inotify()
+        self._move_from_cookies = dict()
 
     @property
     def subscribers(self):
@@ -82,6 +86,41 @@ def __remove_watch(self, folder):
 
         self._notifier.remove_watch(folder.encode(), superficial=True)
 
+    @staticmethod
+    def __is_event_important(event):
+        """
+        :param event: inotify event
+        :return: True if any event type in inotify event is in
+        FileSystemMonitor.important_events list
+        """
+
+        if any(e_type in FileSystemMonitor.important_events
+               for e_type in event[1]):
+            return True
+        return False
+
+    def __gen_events_on_move_to(self, move_to_event):
+        """
+        :param move_to_event: Event object generated for the MOVE_TO event type
+        Notifies subscribers with two events: the first one is about to remove
+        old file, the second is about to create new file
+        """
+
+        if move_to_event.cookie in self._move_from_cookies:
+            move_from_event = self._move_from_cookies[move_to_event.cookie]
+            move_from_event.in_delete = True
+            self.__notify(move_from_event, move_to_event)
+
+    def __notify(self, *events):
+        """
+        :param events: event object(s)
+        Notifies all subscribers about events
+        """
+
+        for subscriber in self._subscribers:
+            for event in events:
+                subscriber.update(event)
+
     def monitor(self):
         """
         Notify subscribers about filesystem events
@@ -92,19 +131,31 @@ def monitor(self):
             self._notifier.add_watch(folder)
 
         for event in self._notifier.event_gen():
-            if event:
-                for subscriber in self._subscribers:
-                    file_object = Event(event)
-                    subscriber.update(file_object)
+            if event and FileSystemMonitor.__is_event_important(event):
+                file_object = Event(event)
+
+                # Saving event object which is being moved in dictionary to
+                if hasattr(file_object, 'IN_MOVED_FROM'):
+                    if file_object.cookie not in self._move_from_cookies:
+                        self._move_from_cookies[file_object.cookie] = \
+                            file_object
+                        continue
+
+                # Generating appropriate events on file movement
+                if hasattr(file_object, 'IN_MOVED_TO'):
+                    self.__gen_events_on_move_to(file_object)
+                    continue
+
+                # Removing a watch on folder 'cause it was deleted
+                if hasattr(file_object, 'IN_DELETE'):
+                    self.__remove_watch(file_object.file_path)
 
-                    # Removing a watch on folder 'cause it was deleted
-                    if hasattr(file_object, 'IN_DELETE'):
-                        self.__remove_watch(file_object.file_path)
+                # Adding a watch on folder 'cause we're being recursive
+                if hasattr(file_object, 'IN_CREATE') and \
+                        os.path.isdir(file_object.file_path):
+                    self.__add_watch(file_object.file_path)
 
-                    # Adding a watch on folder 'cause we're being recursive
-                    if hasattr(file_object, 'IN_CREATE') and \
-                            os.path.isdir(file_object.file_path):
-                        self.__add_watch(file_object.file_path)
+                self.__notify(file_object)
 
 
 # pylint: disable=too-few-public-methods, too-many-instance-attributes
@@ -126,3 +177,5 @@ def __init__(self, event):
         for i_event in type_names:
             if not hasattr(self, i_event):
                 setattr(self, i_event, True)
+        self.is_dir = os.path.isdir(self.file_path)
+        self.in_delete = True if hasattr(self, 'IN_DELETE') else False