From e0258bec3fd84ee6b61f3fde29c283017c184ef5 Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Sun, 19 Nov 2017 12:08:08 -0600 Subject: [PATCH 1/2] add regions and zones --- .../versions/1fdbfd6b0eea_create_instance.py | 2 + .../458762cd0419_create_authn_user.py | 4 +- .../versions/52923fe51ede_create_image.py | 2 + .../versions/9d6460001e00_create_network.py | 1 + .../ba75fca08593_create_regions_and_zones.py | 69 +++++++++++++++++++ .../versions/dadf4ada480a_create_authz.py | 28 ++++++-- ingredients_db/models/images.py | 3 +- ingredients_db/models/instance.py | 4 +- ingredients_db/models/network.py | 3 +- ingredients_db/models/region.py | 40 +++++++++++ ingredients_db/models/zones.py | 43 ++++++++++++ ingredients_db/test/test_migrations.py | 4 ++ 12 files changed, 192 insertions(+), 11 deletions(-) create mode 100644 ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py create mode 100644 ingredients_db/models/region.py create mode 100644 ingredients_db/models/zones.py diff --git a/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py b/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py index 22752e0..a0fff45 100644 --- a/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py +++ b/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py @@ -27,6 +27,8 @@ def upgrade(): sa.Column('tags', HSTORE), sa.Column('state', sa.Enum(InstanceState), default=InstanceState.BUILDING, nullable=False), sa.Column('network_port_id', sau.UUIDType, sa.ForeignKey('network_ports.id', ondelete='RESTRICT')), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), + sa.Column('zone_id', sau.UUIDType, sa.ForeignKey('zones.id')), sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='RESTRICT'), nullable=False), sa.Column('current_task_id', sau.UUIDType, sa.ForeignKey('tasks.id')), diff --git a/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py b/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py index 6dfe591..ad60a66 100644 --- a/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py +++ b/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py @@ -1,7 +1,7 @@ """create authn user Revision ID: 458762cd0419 -Revises: 3ce1572cbc6b +Revises: ba75fca08593 Create Date: 2017-09-16 09:24:55.054833 """ @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = '458762cd0419' -down_revision = '3ce1572cbc6b' +down_revision = 'ba75fca08593' branch_labels = None depends_on = None diff --git a/ingredients_db/alembic/versions/52923fe51ede_create_image.py b/ingredients_db/alembic/versions/52923fe51ede_create_image.py index aa0b1c7..cd88660 100644 --- a/ingredients_db/alembic/versions/52923fe51ede_create_image.py +++ b/ingredients_db/alembic/versions/52923fe51ede_create_image.py @@ -32,6 +32,8 @@ def upgrade(): sa.Column('visibility', sa.Enum(ImageVisibility), default=ImageVisibility.PRIVATE, nullable=False), sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='RESTRICT'), nullable=False), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), + sa.Column('current_task_id', sau.UUIDType, sa.ForeignKey('tasks.id')), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False, index=True), diff --git a/ingredients_db/alembic/versions/9d6460001e00_create_network.py b/ingredients_db/alembic/versions/9d6460001e00_create_network.py index f2c5455..9db0208 100644 --- a/ingredients_db/alembic/versions/9d6460001e00_create_network.py +++ b/ingredients_db/alembic/versions/9d6460001e00_create_network.py @@ -29,6 +29,7 @@ def upgrade(): sa.Column('port_group', sa.String, unique=True, nullable=False), sa.Column('gateway', sau.IPAddressType, nullable=False), sa.Column('dns_servers', sa.ARRAY(sau.IPAddressType), nullable=False), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), sa.Column('state', sa.Enum(NetworkState), default=NetworkState.CREATING, nullable=False), diff --git a/ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py b/ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py new file mode 100644 index 0000000..f396849 --- /dev/null +++ b/ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py @@ -0,0 +1,69 @@ +"""create regions and zones + +Revision ID: ba75fca08593 +Revises: 3ce1572cbc6b +Create Date: 2017-11-04 09:08:59.648307 + +""" +import sqlalchemy as sa +import sqlalchemy_utils as sau +from alembic import op + +# revision identifiers, used by Alembic. +from ingredients_db.models.region import RegionState +from ingredients_db.models.zones import ZoneState + +revision = 'ba75fca08593' +down_revision = '3ce1572cbc6b' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'regions', + sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), + sa.Column('name', sa.String, unique=True, nullable=False), + sa.Column('datacenter', sa.String, unique=True, nullable=False), + sa.Column('image_datastore', sa.String, nullable=False), + sa.Column('image_folder', sa.String), + sa.Column('schedulable', sa.Boolean, nullable=False), + + sa.Column('state', sa.Enum(RegionState), default=RegionState.CREATING, nullable=False), + sa.Column('current_task_id', sau.UUIDType, sa.ForeignKey('tasks.id')), + + sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + nullable=False, index=True), + sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + onupdate=sa.text('clock_timestamp()'), + nullable=False), + + ) + + op.create_table( + 'zones', + sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), + sa.Column('name', sa.String, unique=True, nullable=False), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), + sa.Column('vm_cluster', sa.String, nullable=False), + sa.Column('vm_datastore', sa.String, nullable=False), + sa.Column('vm_folder', sa.String), + sa.Column('core_provision_percent', sa.Integer, nullable=False), + sa.Column('ram_provision_percent', sa.Integer, nullable=False), + sa.Column('schedulable', sa.Boolean, nullable=False), + + sa.Column('state', sa.Enum(ZoneState), default=ZoneState.CREATING, nullable=False), + sa.Column('current_task_id', sau.UUIDType, sa.ForeignKey('tasks.id')), + + sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + nullable=False, index=True), + sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + onupdate=sa.text('clock_timestamp()'), + nullable=False), + + ) + + +def downgrade(): + op.drop_table('zones') + op.drop_table('regions') diff --git a/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py b/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py index 6375b60..0845992 100644 --- a/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py +++ b/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py @@ -106,9 +106,31 @@ def upgrade(): {"name": "roles:list", "rule": "role:admin", "description": "Ability to list roles"}, {"name": "roles:delete", "rule": "role:admin", "description": "Ability to delete a role"}, + # Regions + {"name": "regions:create", "rule": "role:admin", "description": "Ability to create a region"}, + {"name": "regions:get", "rule": "", "description": "Ability to get a region"}, + {"name": "regions:list", "rule": "", "description": "Ability to list regions"}, + {"name": "regions:delete", "rule": "role:admin", "description": "Ability to delete a region"}, + {"name": "regions:action:schedule", "rule": "role:admin", + "description": "Ability to change the schedule mode of the region"}, + + # Zones + {"name": "zones:create", "rule": "role:admin", "description": "Ability to create a zone"}, + {"name": "zones:get", "rule": "", "description": "Ability to get a zone"}, + {"name": "zones:list", "rule": "", "description": "Ability to list zones"}, + {"name": "zones:delete", "rule": "role:admin", "description": "Ability to delete a zone"}, + {"name": "zones:action:schedule", "rule": "role:admin", + "description": "Ability to change the schedule mode of the zone"}, + # Tokens {"name": "tokens:get", "rule": "rule:admin_or_self", "description": "Ability to get a token"}, + # Projects + {"name": "projects:create", "rule": "rule:is_admin", "description": "Ability to create a project"}, + {"name": "projects:get", "rule": "", "description": "Ability to get a project"}, + {"name": "projects:list", "rule": "", "description": "Ability to list projects"}, + {"name": "projects:delete", "rule": "rule:is_admin", "description": "Ability to delete a project"}, + # Tasks # Images @@ -150,12 +172,6 @@ def upgrade(): {"name": "networks:list", "rule": "", "description": "Ability to list networks"}, {"name": "networks:delete", "rule": "rule:is_admin", "description": "Ability to delete a network"}, - # Projects - {"name": "projects:create", "rule": "rule:is_admin", "description": "Ability to create a project"}, - {"name": "projects:get", "rule": "", "description": "Ability to get a project"}, - {"name": "projects:list", "rule": "", "description": "Ability to list projects"}, - {"name": "projects:delete", "rule": "rule:is_admin", "description": "Ability to delete a project"}, - ] ) diff --git a/ingredients_db/models/images.py b/ingredients_db/models/images.py index 0f691a0..2fa0b81 100644 --- a/ingredients_db/models/images.py +++ b/ingredients_db/models/images.py @@ -6,6 +6,7 @@ from ingredients_db.database import Base from ingredients_db.models.project import Project, ProjectMixin +from ingredients_db.models.region import RegionableNixin from ingredients_db.models.task import TaskMixin @@ -36,7 +37,7 @@ class ImageMembers(Base): # TODO: Image families @generic_repr -class Image(Base, TaskMixin, ProjectMixin): +class Image(Base, TaskMixin, ProjectMixin, RegionableNixin): __tablename__ = 'images' id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) diff --git a/ingredients_db/models/instance.py b/ingredients_db/models/instance.py index b4e95af..b2e2821 100644 --- a/ingredients_db/models/instance.py +++ b/ingredients_db/models/instance.py @@ -9,7 +9,9 @@ from ingredients_db.models.network_port import NetworkableMixin from ingredients_db.models.project import ProjectMixin from ingredients_db.models.public_key import PublicKey +from ingredients_db.models.region import RegionableNixin from ingredients_db.models.task import TaskMixin +from ingredients_db.models.zones import ZonableMixin class InstanceState(enum.Enum): @@ -26,7 +28,7 @@ class InstanceState(enum.Enum): @generic_repr -class Instance(Base, TaskMixin, NetworkableMixin, ProjectMixin): +class Instance(Base, TaskMixin, NetworkableMixin, ProjectMixin, RegionableNixin, ZonableMixin): __tablename__ = 'instances' id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) diff --git a/ingredients_db/models/network.py b/ingredients_db/models/network.py index 61d02f4..04cf41e 100644 --- a/ingredients_db/models/network.py +++ b/ingredients_db/models/network.py @@ -8,6 +8,7 @@ from ingredients_db.database import Base from ingredients_db.models.network_port import NetworkPort +from ingredients_db.models.region import RegionableNixin from ingredients_db.models.task import TaskMixin from ingredients_db.types import IPv4Network @@ -21,7 +22,7 @@ class NetworkState(enum.Enum): @generic_repr -class Network(Base, TaskMixin): +class Network(Base, TaskMixin, RegionableNixin): __tablename__ = 'networks' id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) diff --git a/ingredients_db/models/region.py b/ingredients_db/models/region.py new file mode 100644 index 0000000..95c8d89 --- /dev/null +++ b/ingredients_db/models/region.py @@ -0,0 +1,40 @@ +import enum + +from sqlalchemy import Column, text, String, Boolean, Enum, ForeignKey +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy_utils import generic_repr, UUIDType, ArrowType + +from ingredients_db.database import Base +from ingredients_db.models.task import TaskMixin + + +class RegionState(enum.Enum): + CREATING = 'CREATING' + CREATED = 'CREATED' + DELETING = 'DELETING' + DELETED = 'DELETED' + ERROR = 'ERROR' + + +@generic_repr +class Region(Base, TaskMixin): + __tablename__ = 'regions' + + id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) + name = Column(String, unique=True, nullable=False) + datacenter = Column(String, unique=True, nullable=False) + image_datastore = Column(String, nullable=False) + image_folder = Column(String) + schedulable = Column(Boolean, nullable=False) + + state = Column(Enum(RegionState), default=RegionState.CREATING, nullable=False) + + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) + updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), + onupdate=text('clock_timestamp()'), nullable=False) + + +class RegionableNixin(object): + @declared_attr + def region_id(cls): + return Column(UUIDType, ForeignKey('regions.id', ondelete='RESTRICT'), nullable=False) diff --git a/ingredients_db/models/zones.py b/ingredients_db/models/zones.py new file mode 100644 index 0000000..2bb33ee --- /dev/null +++ b/ingredients_db/models/zones.py @@ -0,0 +1,43 @@ +import enum + +from sqlalchemy import Column, text, String, Boolean, ForeignKey, Integer, Enum +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy_utils import generic_repr, UUIDType, ArrowType + +from ingredients_db.database import Base +from ingredients_db.models.region import RegionableNixin +from ingredients_db.models.task import TaskMixin + + +class ZoneState(enum.Enum): + CREATING = 'CREATING' + CREATED = 'CREATED' + DELETING = 'DELETING' + DELETED = 'DELETED' + ERROR = 'ERROR' + + +@generic_repr +class Zone(Base, TaskMixin, RegionableNixin): + __tablename__ = 'zones' + + id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) + name = Column(String, unique=True, nullable=False) + vm_cluster = Column(String, nullable=False) + vm_datastore = Column(String, nullable=False) + vm_folder = Column(String) + core_provision_percent = Column(Integer, nullable=False) + ram_provision_percent = Column(Integer, nullable=False) + schedulable = Column(Boolean, nullable=False) + + state = Column(Enum(ZoneState), default=ZoneState.CREATING, nullable=False) + + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) + updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), + onupdate=text('clock_timestamp()'), nullable=False) + + +class ZonableMixin(object): + @declared_attr + def zone_id(cls): + return Column(UUIDType, ForeignKey('zones.id', ondelete='RESTRICT')) diff --git a/ingredients_db/test/test_migrations.py b/ingredients_db/test/test_migrations.py index 0c544df..ec71b94 100644 --- a/ingredients_db/test/test_migrations.py +++ b/ingredients_db/test/test_migrations.py @@ -79,6 +79,8 @@ def test_model_and_migration_schemas_are_the_same(self, uri_left, uri_right, ale from ingredients_db.models.task import Task from ingredients_db.models.authn import AuthNUser, AuthNToken from ingredients_db.models.authz import AuthZPolicy, AuthZRole + from ingredients_db.models.region import Region + from ingredients_db.models.zones import Zone # Make sure the imports don't go away Image.mro() @@ -94,6 +96,8 @@ def test_model_and_migration_schemas_are_the_same(self, uri_left, uri_right, ale AuthNToken.mro() AuthZPolicy.mro() AuthZRole.mro() + Region.mro() + Zone.mro() setup_extensions(uri_right) prepare_schema_from_models(uri_right, Base) From ee26c837afb025366d7c25bea07ea01851cd9b63 Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Sun, 19 Nov 2017 12:13:05 -0600 Subject: [PATCH 2/2] fix migrations --- .../alembic/versions/1fdbfd6b0eea_create_instance.py | 4 ++-- ingredients_db/alembic/versions/52923fe51ede_create_image.py | 2 +- .../alembic/versions/9d6460001e00_create_network.py | 2 +- .../alembic/versions/ba75fca08593_create_regions_and_zones.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py b/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py index a0fff45..40c4e73 100644 --- a/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py +++ b/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py @@ -27,8 +27,8 @@ def upgrade(): sa.Column('tags', HSTORE), sa.Column('state', sa.Enum(InstanceState), default=InstanceState.BUILDING, nullable=False), sa.Column('network_port_id', sau.UUIDType, sa.ForeignKey('network_ports.id', ondelete='RESTRICT')), - sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), - sa.Column('zone_id', sau.UUIDType, sa.ForeignKey('zones.id')), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id', ondelete='RESTRICT'), nullable=False), + sa.Column('zone_id', sau.UUIDType, sa.ForeignKey('zones.id', ondelete='RESTRICT')), sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='RESTRICT'), nullable=False), sa.Column('current_task_id', sau.UUIDType, sa.ForeignKey('tasks.id')), diff --git a/ingredients_db/alembic/versions/52923fe51ede_create_image.py b/ingredients_db/alembic/versions/52923fe51ede_create_image.py index cd88660..b179e8a 100644 --- a/ingredients_db/alembic/versions/52923fe51ede_create_image.py +++ b/ingredients_db/alembic/versions/52923fe51ede_create_image.py @@ -32,7 +32,7 @@ def upgrade(): sa.Column('visibility', sa.Enum(ImageVisibility), default=ImageVisibility.PRIVATE, nullable=False), sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='RESTRICT'), nullable=False), - sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id', ondelete='RESTRICT'), nullable=False), sa.Column('current_task_id', sau.UUIDType, sa.ForeignKey('tasks.id')), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), diff --git a/ingredients_db/alembic/versions/9d6460001e00_create_network.py b/ingredients_db/alembic/versions/9d6460001e00_create_network.py index 9db0208..6671936 100644 --- a/ingredients_db/alembic/versions/9d6460001e00_create_network.py +++ b/ingredients_db/alembic/versions/9d6460001e00_create_network.py @@ -29,7 +29,7 @@ def upgrade(): sa.Column('port_group', sa.String, unique=True, nullable=False), sa.Column('gateway', sau.IPAddressType, nullable=False), sa.Column('dns_servers', sa.ARRAY(sau.IPAddressType), nullable=False), - sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id', ondelete='RESTRICT'), nullable=False), sa.Column('state', sa.Enum(NetworkState), default=NetworkState.CREATING, nullable=False), diff --git a/ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py b/ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py index f396849..0855481 100644 --- a/ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py +++ b/ingredients_db/alembic/versions/ba75fca08593_create_regions_and_zones.py @@ -44,7 +44,7 @@ def upgrade(): 'zones', sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), sa.Column('name', sa.String, unique=True, nullable=False), - sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id'), nullable=False), + sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id', ondelete='RESTRICT'), nullable=False), sa.Column('vm_cluster', sa.String, nullable=False), sa.Column('vm_datastore', sa.String, nullable=False), sa.Column('vm_folder', sa.String),