Merge pull request #395 from HTTP-APIs/develop

Author: Mec-iS

cli.py

@@ -3,7 +3,8 @@
 
 from hydrus.app_factory import app_factory
 from hydrus.utils import (set_session, set_doc, set_hydrus_server_url,
-                          set_token, set_api_name, set_authentication)
+                          set_token, set_api_name, set_authentication,
+                          set_page_size, set_pagination)
 from hydrus.data import doc_parse
 from hydra_python_core import doc_maker
 from hydrus.data.db_models import Base
@@ -33,13 +34,17 @@
               type=str)
 @click.option("--port", "-p", default=8080,
               help="The port the app is hosted at.", type=int)
+@click.option("--pagesize", "-ps", default=10,
+              help="Maximum size of a page(view)", type=int)
+@click.option("--pagination/--no-pagination", default=True,
+              help="Enable or disable pagination.")
 @click.option("--token/--no-token", default=True,
               help="Toggle token based user authentication.")
 @click.option("--serverurl", default="http://localhost",
               help="Set server url", type=str)
 @click.argument("serve", required=True)
-def startserver(adduser: Tuple, api: str, auth: bool, dburl: str,
-                hydradoc: str, port: int, serverurl: str, token: bool,
+def startserver(adduser: Tuple, api: str, auth: bool, dburl: str, pagination: bool,
+                hydradoc: str, port: int, pagesize: int, serverurl: str, token: bool,
                 serve: None) -> None:
     """
     Python Hydrus CLI
@@ -165,17 +170,21 @@ def startserver(adduser: Tuple, api: str, auth: bool, dburl: str,
                     with set_hydrus_server_url(app, HYDRUS_SERVER_URL):
                         # Set the Database session
                         with set_session(app, session):
-                            # Start the Hydrus app
-                            http_server = WSGIServer(('', port), app)
-                            click.echo("Server running at:")
-                            click.echo(
-                                "{}{}".format(
-                                    HYDRUS_SERVER_URL,
-                                    API_NAME))
-                            try:
-                                http_server.serve_forever()
-                            except KeyboardInterrupt:
-                                pass
+                            # Enable/disable pagination
+                            with set_pagination(app, pagination):
+                                # Set page size of a collection view
+                                with set_page_size(app, pagesize):
+                                    # Start the hydrus app
+                                    http_server = WSGIServer(('', port), app)
+                                    click.echo("Server running at:")
+                                    click.echo(
+                                        "{}{}".format(
+                                            HYDRUS_SERVER_URL,
+                                            API_NAME))
+                                    try:
+                                        http_server.serve_forever()
+                                    except KeyboardInterrupt:
+                                        pass
 
 
 if __name__ == "__main__":

hydrus/data/crud.py

@@ -43,7 +43,8 @@
     PropertyNotFound,
     NotInstanceProperty,
     NotAbstractProperty,
-    InstanceNotFound)
+    InstanceNotFound,
+    PageNotFound)
 # from sqlalchemy.orm.session import Session
 from sqlalchemy.orm.scoping import scoped_session
 from typing import Dict, Optional, Any, List
@@ -514,19 +515,30 @@ def update(id_: str,
 def get_collection(API_NAME: str,
                    type_: str,
                    session: scoped_session,
-                   path: str = None) -> Dict[str,
-                                             Any]:
+                   paginate: bool,
+                   page_size: int,
+                   page: int = 1,
+                   path: str = None) -> Dict[str, Any]:
     """Retrieve a type of collection from the database.
     :param API_NAME: api name specified while starting server
     :param type_: type of object to be updated
     :param session: sqlalchemy scoped session
+    :param paginate: Enable/disable pagination
+    :param page_size: Number maximum elements showed in a page
+    :param page: page number
     :param path: endpoint
-    :return: response containing all the objects of that particular type_
+    :return: response containing a page of the objects of that particular type_
 
     Raises:
-        ClassNotFound: If `type_` does not represt a valid/defined RDFClass.
+        ClassNotFound: If `type_` does not represent a valid/defined RDFClass.
 
     """
+    # Check for valid page value
+    try:
+        page = int(page)
+    except ValueError:
+        raise PageNotFound(page)
+
     if path is not None:
         collection_template = {
             "@id": "/{}/{}/".format(API_NAME, path),
@@ -548,8 +560,13 @@ def get_collection(API_NAME: str,
         raise ClassNotFound(type_=type_)
 
     try:
-        instances = session.query(Instance).filter(
-            Instance.type_ == rdf_class.id).all()
+        if paginate is True:
+            offset = (page - 1) * page_size
+            instances = session.query(Instance).filter(
+                Instance.type_ == rdf_class.id).limit(page_size).offset(offset)
+        else:
+            instances = session.query(Instance).filter(
+                Instance.type_ == rdf_class.id).all()
     except NoResultFound:
         instances = list()
 
@@ -563,6 +580,37 @@ def get_collection(API_NAME: str,
             object_template = {
                 "@id": "/{}/{}Collection/{}".format(API_NAME, type_, instance_.id), "@type": type_}
         collection_template["members"].append(object_template)
+
+    # If pagination is disabled then stop and return the collection template
+    if paginate is False:
+        return collection_template
+
+    number_of_instances = len(collection_template["members"])
+    # If we are on the first page and there are fewer elements than the
+    # page size then there is no need to make an extra DB call to get count
+    if page == 1 and number_of_instances < page_size:
+        total_items = number_of_instances
+    else:
+        total_items = session.query(Instance).filter(
+            Instance.type_ == rdf_class.id).count()
+    collection_template["totalItems"] = total_items
+    # Calculate last page number
+    if total_items != 0 and total_items % page_size == 0:
+        last = total_items // page_size
+    else:
+        last = total_items // page_size + 1
+    if page < 1 or page > last:
+        raise PageNotFound(str(page))
+    collection_template["view"] = {
+        "@id": "/{}/{}?page={}".format(API_NAME, path, page),
+        "@type": "PartialCollectionView",
+        "first": "/{}/{}?page=1".format(API_NAME, path),
+        "last": "/{}/{}?page={}".format(API_NAME, path, last)
+    }
+    if page != 1:
+        collection_template["view"]["previous"] = "/{}/{}?page={}".format(API_NAME, path, page-1)
+    if page != last:
+        collection_template["view"]["next"] = "/{}/{}?page={}".format(API_NAME, path, page + 1)
     return collection_template
 
 

hydrus/data/exceptions.py

@@ -114,3 +114,16 @@ def get_HTTP(self) -> Tuple[int, Dict[str, str]]:
         """Return the HTTP response for the Exception."""
         return 400, {
             "message": "The User with ID {} is not a valid/defined User".format(self.id_)}
+
+
+class PageNotFound(Exception):
+    """Error when the User is not found."""
+
+    def __init__(self, page_id: str) -> None:
+        """Constructor."""
+        self.page_id = page_id
+
+    def get_HTTP(self) -> Tuple[int, Dict[str, str]]:
+        """Return the HTTP response for the Exception."""
+        return 400, {
+            "message": "The page with ID {} not found".format(self.page_id)}

hydrus/resources.py

@@ -39,7 +39,8 @@
     ClassNotFound,
     InstanceExists,
     PropertyNotFound,
-    InstanceNotFound)
+    InstanceNotFound,
+    PageNotFound)
 from hydrus.helpers import (
     set_response_headers,
     checkClassOp,
@@ -52,7 +53,12 @@
     check_read_only_props,
     check_required_props,
     finalize_response)
-from hydrus.utils import get_session, get_doc, get_api_name, get_hydrus_server_url
+from hydrus.utils import (get_session,
+    get_doc,
+    get_api_name,
+    get_hydrus_server_url,
+    get_page_size,
+    get_pagination)
 
 
 class Index(Resource):
@@ -226,6 +232,7 @@ def get(self, path: str) -> Response:
         """
         Retrieve a collection of items from the database.
         """
+        page = request.args.get('page')
         auth_response = check_authentication_response()
         if isinstance(auth_response, Response):
             return auth_response
@@ -239,11 +246,24 @@ def get(self, path: str) -> Response:
             collection = get_doc().collections[path]["collection"]
             try:
                 # Get collection details from the database
-                response = crud.get_collection(
-                    get_api_name(), collection.class_.title, session=get_session(), path=path)
+                if get_pagination():
+                    # Get paginated response
+                    if page is None:
+                        response = crud.get_collection(
+                            get_api_name(), collection.class_.title, session=get_session(),
+                            paginate=True, page_size=get_page_size(), path=path)
+                    else:
+                        response = crud.get_collection(
+                            get_api_name(), collection.class_.title, session=get_session(),
+                            paginate=True, page=page, page_size=get_page_size(), path=path)
+                else:
+                    # Get whole collection
+                    response = crud.get_collection(
+                        get_api_name(), collection.class_.title, session=get_session(),
+                        paginate=False, path=path)
                 return set_response_headers(jsonify(hydrafy(response, path=path)))
 
-            except ClassNotFound as e:
+            except (ClassNotFound, PageNotFound) as e:
                 status_code, message = e.get_HTTP()
                 return set_response_headers(jsonify(message), status_code=status_code)
 

hydrus/tests/test_app.py

@@ -6,7 +6,7 @@
 import re
 import uuid
 from hydrus.app_factory import app_factory
-from hydrus.utils import set_session, set_doc, set_api_name
+from hydrus.utils import set_session, set_doc, set_api_name, set_page_size
 from hydrus.data import doc_parse, crud
 from hydra_python_core import doc_maker
 from hydrus.samples import doc_writer_sample
@@ -44,6 +44,7 @@ def setUpClass(self):
 
         self.session = session
         self.API_NAME = "demoapi"
+        self.page_size = 1
         self.HYDRUS_SERVER_URL = "http://hydrus.com/"
 
         self.app = app_factory(self.API_NAME)
@@ -60,10 +61,11 @@ def setUpClass(self):
 
         print("Classes and properties added successfully.")
 
-        print("Setting up Hydrus utilities... ")
+        print("Setting up hydrus utilities... ")
         self.api_name_util = set_api_name(self.app, self.API_NAME)
         self.session_util = set_session(self.app, self.session)
         self.doc_util = set_doc(self.app, self.doc)
+        self.page_size_util = set_page_size(self.app, self.page_size)
         self.client = self.app.test_client()
 
         print("Creating utilities context... ")
@@ -85,8 +87,14 @@ def tearDownClass(self):
 
     def setUp(self):
         for class_ in self.doc.parsed_classes:
-            if class_ not in self.doc.collections:
-                dummy_obj = gen_dummy_object(class_, self.doc)
+            dummy_obj = gen_dummy_object(class_, self.doc)
+            crud.insert(
+                dummy_obj,
+                id_=str(
+                    uuid.uuid4()),
+                session=self.session)
+            # If it's a collection class then add an extra object so we can test pagination thoroughly.
+            if class_ in self.doc.collections:
                 crud.insert(
                     dummy_obj,
                     id_=str(
@@ -166,6 +174,30 @@ def test_Collections_GET(self):
                 assert "@type" in response_get_data
                 assert "members" in response_get_data
 
+    def test_pagination(self):
+        """Test basic pagination"""
+        index = self.client.get("/{}".format(self.API_NAME))
+        assert index.status_code == 200
+        endpoints = json.loads(index.data.decode('utf-8'))
+        for endpoint in endpoints:
+            collection_name = "/".join(endpoints[endpoint].split(
+                "/{}/".format(self.API_NAME))[1:])
+            if collection_name in self.doc.collections:
+                response_get = self.client.get(endpoints[endpoint])
+                assert response_get.status_code == 200
+                response_get_data = json.loads(
+                    response_get.data.decode('utf-8'))
+                assert "view" in response_get_data
+                assert "first" in response_get_data["view"]
+                assert "last" in response_get_data["view"]
+                if "next" in response_get_data["view"]:
+                    response_next = self.client.get(response_get_data["view"]["next"])
+                    assert response_next.status_code == 200
+                    response_next_data = json.loads(
+                        response_next.data.decode('utf-8'))
+                    assert "previous" in response_next_data["view"]
+                break
+
     def test_Collections_PUT(self):
         """Test insert data to the collection."""
         index = self.client.get("/{}".format(self.API_NAME))