add postgresql support to docker, try catch for postgresql

Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER binux <[email protected]>
 # install python
 RUN apt-get update && \
         apt-get install -y python python-dev python-distribute python-pip && \
-        apt-get install -y libcurl4-openssl-dev libxml2-dev libxslt1-dev python-lxml python-mysqldb
+        apt-get install -y libcurl4-openssl-dev libxml2-dev libxslt1-dev python-lxml python-mysqldb libpq-dev
 
 # install requirements
 ADD requirements.txt /opt/pyspider/requirements.txt

pyspider/database/sqlalchemy/projectdb.py

@@ -7,6 +7,7 @@
 
 import six
 import time
+import sqlalchemy.exc
 
 from sqlalchemy import create_engine, MetaData, Table, Column, String, Float, Text
 from sqlalchemy.engine.url import make_url
@@ -41,8 +42,11 @@ def __init__(self, url):
         if self.url.database:
             database = self.url.database
             self.url.database = None
-            engine = create_engine(self.url, convert_unicode=False)
-            engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
+            try:
+                engine = create_engine(self.url, convert_unicode=False)
+                engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
+            except sqlalchemy.exc.OperationalError:
+                pass
             self.url.database = database
         self.engine = create_engine(url, convert_unicode=False)
         self.table.create(self.engine, checkfirst=True)

pyspider/database/sqlalchemy/resultdb.py

@@ -9,6 +9,7 @@
 import six
 import time
 import json
+import sqlalchemy.exc
 
 from sqlalchemy import (create_engine, MetaData, Table, Column,
                         String, Float, LargeBinary)
@@ -40,8 +41,11 @@ def __init__(self, url):
         if self.url.database:
             database = self.url.database
             self.url.database = None
-            engine = create_engine(self.url, convert_unicode=True)
-            engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
+            try:
+                engine = create_engine(self.url, convert_unicode=True)
+                engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
+            except sqlalchemy.exc.OperationalError:
+                pass
             self.url.database = database
         self.engine = create_engine(url, convert_unicode=True)
 

pyspider/database/sqlalchemy/taskdb.py

@@ -9,6 +9,7 @@
 import six
 import time
 import json
+import sqlalchemy.exc
 
 from sqlalchemy import (create_engine, MetaData, Table, Column, Index,
                         Integer, String, Float, LargeBinary, func)
@@ -46,8 +47,11 @@ def __init__(self, url):
         if self.url.database:
             database = self.url.database
             self.url.database = None
-            engine = create_engine(self.url, convert_unicode=True)
-            engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
+            try:
+                engine = create_engine(self.url, convert_unicode=True)
+                engine.execute("CREATE DATABASE IF NOT EXISTS %s" % database)
+            except sqlalchemy.exc.OperationalError:
+                pass
             self.url.database = database
         self.engine = create_engine(self.url, convert_unicode=True)
 

requirements.txt

@@ -19,3 +19,4 @@ six
 amqp>=1.3.0
 redis
 kombu
+psycopg2

setup.py

@@ -44,6 +44,7 @@
     'SQLAlchemy>=0.9.7',
     'redis',
     'kombu',
+    'psycopg2',
 ]
 if sys.version_info < (3, 0):
     extras_require_all.extend([