diff --git a/README.md b/README.md index 7017d56..3fcce90 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,74 @@ Velocity Works Coding Demo # Golang-Demo -This Golang application consumes a JSON payload from https://www.data.gov/, populates a database and displays the database contents on a web page. +This Golang application consumes a JSON payload from https://www.data.gov/ and populates a database to display the database contents on a web page. -Frameworks used: +It is a simple http server to donwload the JSON from https://labs.data.gov/dashboard/offices/qa . It features Rest endpoint to get the JSON response & save in a database. It is written using Echo Web Framework to make server high performant. -- Echo to build the website: https://github.com/labstack/echo -- Resty to retrieve data from Data.gov: https://github.com/go-resty/resty -- Jsonparser to process the data: https://github.com/buger/jsonparser -- GORM to populate the database: https://github.com/jinzhu/gorm + +## Contents + +- [Golang-Demo](#golang-demo) + - [Usage](#usage) + - [Frameworks](#frameworks) + - [Performance Metrics](#performance-metrics) + - [Limitations](#limitations) + + +## Usage + +To install Golang-Demo, you need to install [Go](https://golang.org/)(**version 1.12+ is required**) and set your Go workspace. + +1. This project uses go modules +2. This project has makefile. You should be able to simply install and start: + +```sh +$ git clone https://github.com/anil-appface/golang-demo.git +$ cd golang-demo +$ make +$ ./go-datagov +``` + + +## Frameworks + +This project uses the below frameworks: + +1. Echo Framework: Simple & high performant server +2. Resty: Simple HTTP helper to get information +3. JSON parser: To process the JSON data +4. Gorm: To populate the database + +## Performance Metrics + +Benchmarking for this application is not done. + +

"As this application uses Echo web framework, the default logs of echo server shows the Method type, uri, and Status code(Which is configurable in main.go). Also it shows the logging of method name, line number and file."
+ + +{"time":"2021-02-04T11:48:34.559638168+05:30","id":"","remote_ip":"::1","host":"localhost:8000","method":"GET","uri":"/data?url=https://www.defense.gov/data.json","user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36","status":200,"error":"","latency":3516046682,"latency_human":"3.516046682s","bytes_in":0,"bytes_out":294234} + +

+ + +## Limitations + +1. There is no authentication wrapper around the API's. +2. There is no field validation in api. +4. There is no much test cases written in the interest of time. +5. There is no cron/automation to scrape the defined URL response to store in DB + +## Rest API's + + +1. Web page + http://localhost:8000 + +2. To get JSON response from DB: + http://localhost:8000/data?url=https://www.defense.gov/data.json + + url query param is optional. If you do not pass url in the query, then default first record will be returned. + +### API structure + + diff --git a/docs/api-setup.jpg b/docs/api-setup.jpg new file mode 100644 index 0000000..e24dcc0 Binary files /dev/null and b/docs/api-setup.jpg differ diff --git a/go-datagov b/go-datagov new file mode 100755 index 0000000..596622c Binary files /dev/null and b/go-datagov differ diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e21366a --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module github.com/anil-appface/golang-demo + +go 1.15 + +require ( + github.com/buger/jsonparser v1.1.1 + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/go-resty/resty/v2 v2.4.0 + github.com/jinzhu/gorm v1.9.16 + github.com/labstack/echo v3.3.10+incompatible + github.com/labstack/gommon v0.3.0 + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-sqlite3 v1.14.6 // indirect + github.com/valyala/fasttemplate v1.2.1 // indirect + golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect + golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect + gorm.io/driver/sqlite v1.1.4 // indirect + gorm.io/gorm v1.20.12 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e14accb --- /dev/null +++ b/go.sum @@ -0,0 +1,92 @@ +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v1.0.2 h1:KPldsxuKGsS2FPWsNeg9ZO18aCrGKujPoWXn2yo+KQM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/go-resty/resty v1.12.0 h1:L1P5qymrXL5H/doXe2pKUr1wxovAI5ilm2LdVLbwThc= +github.com/go-resty/resty/v2 v2.4.0 h1:s6TItTLejEI+2mn98oijC5w/Rk2YU+OA6x0mnZN6r6k= +github.com/go-resty/resty/v2 v2.4.0/go.mod h1:B88+xCTEwvfD94NOuE6GS1wMlnoKNY8eEiNizfNwOwA= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= +github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI= +github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg= +github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= +github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= +github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= +gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= +gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.12 h1:ebZ5KrSHzet+sqOCVdH9mTjW91L298nX3v5lVxAzSUY= +gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= diff --git a/main.go b/main.go new file mode 100644 index 0000000..ef0ac11 --- /dev/null +++ b/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "github.com/anil-appface/golang-demo/restHandlers" + + _ "github.com/jinzhu/gorm/dialects/sqlite" +) + +func main() { + + //initialising new server + srv := restHandlers.NewServer() + + //start the http server along with dependencies + srv.Start() +} diff --git a/makefile b/makefile new file mode 100644 index 0000000..168de47 --- /dev/null +++ b/makefile @@ -0,0 +1,2 @@ +all: + env GO111MODULE=on GOOS=linux GOARCH=amd64 go build -o go-datagov . \ No newline at end of file diff --git a/restHandlers/dataHandler.go b/restHandlers/dataHandler.go new file mode 100644 index 0000000..58d820c --- /dev/null +++ b/restHandlers/dataHandler.go @@ -0,0 +1,109 @@ +package restHandlers + +import ( + "net/http" + "time" + + "github.com/anil-appface/golang-demo/store" + "github.com/go-resty/resty/v2" + "github.com/jinzhu/gorm" + "github.com/labstack/echo" +) + +type dataHandler struct { + _client *resty.Client + _db *gorm.DB +} + +//NewDataHandler creates the new instance for dataHandler +func NewDataHandler(client *resty.Client, db *gorm.DB) *dataHandler { + return &dataHandler{ + _client: client, + _db: db, + } +} + +//Get request gets the index page +func (me *dataHandler) Get(c echo.Context) error { + return c.Render(http.StatusOK, "index.html", nil) +} + +// Info gets the catalog information & will present in the context +func (me *dataHandler) Info(c echo.Context) error { + + var err error + catalog := &store.Catalog{} + url := c.FormValue("urldetails") + if url != "" { + catalog, err = me.saveAndGetData(c, url) + if err != nil { + return err + } + } + + return c.Render(http.StatusOK, "info.html", catalog) + +} + +//GetData returns the data in JSON format +func (me *dataHandler) GetData(c echo.Context) error { + + var err error + catalog := &store.Catalog{} + url := c.QueryParam("url") + if url != "" { + catalog, err = me.saveAndGetData(c, url) + if err != nil { + return err + } + } else { + //Get the first record + catalog.First(me._db) + } + + return c.JSONPretty(http.StatusOK, catalog, "\t") +} + +//performDBaction downlaods data from specific URL and saves it inside db if data is not older than a day +func (me *dataHandler) saveAndGetData(c echo.Context, url string) (*store.Catalog, error) { + + //cleanup if the data for the given url is already exists + catalog := &store.Catalog{} + me._db.Where("url = ?", url).First(catalog) + + //Delete items & save if the response is not saved + if catalog.URL == "" || time.Now().Sub(catalog.CreatedAt) >= 24*time.Hour { + + //Delete the catalog + if catalog.URL != "" { + catalog.Delete(me._db) + } + + //make request + c.Logger().Infof("making request url: %s", url) + resp, err := me._client.R().Get(url) + if err != nil { + return nil, err + } + //read response + catalog = &store.Catalog{} + c.Logger().Info("parsing the response to catalog") + catalog.URL = url + err = catalog.Parse(resp.Body()) + if err != nil { + return nil, err + } + + //saving to database + c.Logger().Info("Storing to database") + if err = me._db.Create(&catalog).Error; err != nil { + return nil, err + } + } else { + + //fetch all datasets + catalog.GetDatasets(me._db) + } + + return catalog, nil +} diff --git a/restHandlers/server.go b/restHandlers/server.go new file mode 100644 index 0000000..bfa09a3 --- /dev/null +++ b/restHandlers/server.go @@ -0,0 +1,112 @@ +package restHandlers + +import ( + "context" + "html/template" + "os" + "os/signal" + "syscall" + "time" + + "github.com/anil-appface/golang-demo/store" + "github.com/anil-appface/golang-demo/utils" + "github.com/go-resty/resty/v2" + "github.com/jinzhu/gorm" + "github.com/labstack/echo" + "github.com/labstack/echo/middleware" + "github.com/labstack/gommon/log" +) + +const dataGovURL = "https://www.consumerfinance.gov/data.json" + +type Server struct { + _e *echo.Echo + _client *resty.Client + _db *gorm.DB +} + +//NewServer creates new server +func NewServer() *Server { + + //create a new echo + e := echo.New() + + //create a new resty + c := resty.New() + + //initialise db + db, err := store.OpenDBconnection() + if err != nil { + panic(err) + } + //setup template + e.Renderer = &utils.Template{ + Template: template.Must(template.ParseGlob("static/*.html")), + } + + return &Server{ + _e: e, + _client: c, + _db: db, + } +} + +//Start initialise the prerequisites to start the server +func (me *Server) Start() { + + //setup routers. + me.setupRouters() + + logger := log.New("") + + logger.SetHeader("[${time_rfc3339}] ${level} | ${short_file} | line=${line} |") + + me._client.SetLogger(logger) + me._e.Logger = logger + + //setup default logger middleware + me._e.Use(middleware.Logger()) + + //start the server with graceful shutdown + me.Run() + +} + +// Run will run the HTTP Server +func (me *Server) Run() { + // Set up a channel to listen to for interrupt signals + var runChan = make(chan os.Signal, 1) + + // Set up a context to allow for graceful server shutdowns in the event + // of an OS interrupt (defers the cancel just in case) + _, cancel := context.WithTimeout( + context.Background(), + time.Minute*30, + ) + defer cancel() + + // Handle ctrl+c/ctrl+x interrupt + signal.Notify(runChan, os.Interrupt, syscall.SIGTSTP) + + // Run the server on a new goroutine + go func() { + if err := me._e.Start(":8000"); err != nil { + log.Fatalf("Server failed to start due to err: %v", err) + } + }() + + // Block on this channel listeninf for those previously defined syscalls assign + // to variable so we can let the user know why the server is shutting down + interrupt := <-runChan + + // If we get one of the pre-prescribed syscalls, gracefully terminate the server + // while alerting the user + log.Printf("Server is shutting down due to %+v\n", interrupt) +} + +func (me *Server) setupRouters() { + dh := NewDataHandler(me._client, me._db) + me._e.GET("/", dh.Get) + me._e.POST("/info", dh.Info) + me._e.GET("/data", dh.GetData) +} diff --git a/restHandlers/server_test.go b/restHandlers/server_test.go new file mode 100644 index 0000000..e8652ca --- /dev/null +++ b/restHandlers/server_test.go @@ -0,0 +1,46 @@ +package restHandlers + +import ( + "testing" + + "github.com/anil-appface/golang-demo/store" + "github.com/go-resty/resty/v2" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/sqlite" + "github.com/labstack/echo" +) + +func TestPopulateDB(t *testing.T) { + + e := echo.New() //create a new echo + c := resty.New() //create a new resty + db, err := testDBconnection() + if err != nil { + panic(err) + } + dataHandler := NewDataHandler(c, db) + //url := "https://www.consumerfinance.gov/data.json" + url := "https://www.defense.gov/data.json" + catalog, err := dataHandler.saveAndGetData(e.AcquireContext(), url) + if err != nil { + t.Fail() + } + + if catalog.URL == "" { + t.Fail() + } +} + +//To open & setup db connection +func testDBconnection() (*gorm.DB, error) { + db, err := gorm.Open("sqlite3", "velocityworks_test.db") + if err != nil { + return nil, err + } + err = db.AutoMigrate(&store.Catalog{}, &store.Distribution{}, &store.Publisher{}, + &store.ContactPoint{}, &store.Dataset{}).Error + if err != nil { + return nil, err + } + return db, nil +} diff --git a/static/index.html b/static/index.html new file mode 100644 index 0000000..e311e0d --- /dev/null +++ b/static/index.html @@ -0,0 +1,54 @@ + + + + + Velocity Works - data.gov + + + +

Welcome to data.gov search

+
+
+ This application supports the all the URLS from the below site: +

+ https://labs.data.gov/dashboard/offices/qa +

+
+ +
+ +
+ +
+
+ + diff --git a/static/info.html b/static/info.html new file mode 100644 index 0000000..1d2fc3a --- /dev/null +++ b/static/info.html @@ -0,0 +1,109 @@ + + + + + data.gov + + + +
+

Data.gov information

+
+
+
+
+
+ {{.ID}} - {{.URL}} +
+ Conforms to + Described By + Context +
+
+
+ + {{range .Dataset}} +
+

{{.Title}}

+

Description: {{.Description}}

+ + + + + + + + + + + + + + + + + + + +
ModifiedAccess LevelIdentifierLicenseKeywordsBureau CodesProgram Codes
{{.Modified}}{{.AccessLevel}}{{.Identifier}}{{.License}}{{.Keywords}}{{.BureauCodes}}{{.ProgramCodes}}
+ + {{if .Distributions}} +

Distributions:

+ + + + + + + + + + {{range .Distributions}} + + + + + + + + {{end}} +
MediaTypeConformsToFormatAccess URLDownload URL
{{.MediaType}}{{.ConformsTo}}{{.Format}}{{.AccessURL}}{{.DownloadURL}}
+ + {{end}} {{with .Publisher}} +
+

Name: {{.Name}}

+
+ {{end}} {{with .ContactPoint}} +
+

Contact: {{.Fn}}

+
+ {{end}} +
+ + {{end}} +
+
+ + diff --git a/store/catalog.go b/store/catalog.go new file mode 100644 index 0000000..e23e132 --- /dev/null +++ b/store/catalog.go @@ -0,0 +1,87 @@ +package store + +import ( + "github.com/buger/jsonparser" + "github.com/jinzhu/gorm" +) + +// Catalog - first level in the payload hierarchy. +type Catalog struct { + gorm.Model + + URL string `json:"url,omitempty"` + ConformsTo string `json:"conformsTo,omitempty"` + DescribedBy string `json:"describedBy,omitempty"` + Context string `json:"context,omitempty"` + MetadataType string `json:"@type,omitempty"` + Dataset []Dataset `json:"dataset,omitempty"` +} + +// Parse json response. +func (me *Catalog) Parse(data []byte) error { + + var err error + + if me.MetadataType, err = jsonparser.GetString(data, "@type"); err != nil { + return err + } + + me.ConformsTo, _ = jsonparser.GetString(data, "conformsTo") + me.DescribedBy, _ = jsonparser.GetString(data, "describedBy") + me.Context, _ = jsonparser.GetString(data, "@context") + + datasets, dataType, _, _ := jsonparser.Get(data, "dataset") + + if dataType == jsonparser.Array { + _, err = jsonparser.ArrayEach( + datasets, + func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + dataset := &Dataset{} + if err = dataset.Parse(value); err != nil { + return + } + if dataset != nil { + me.Dataset = append(me.Dataset, *dataset) + } + }) + if err != nil { + return err + } + } + + return nil +} + +//First Get first record +func (me *Catalog) First(db *gorm.DB) { + db.First(me) + db.Where("catalog_id=?", me.ID).Find(&me.Dataset) + for i := 0; i < len(me.Dataset); i++ { + db.Where("dataset_id=?", me.Dataset[i].ID).Find(&me.Dataset[i].Publisher) + db.Where("dataset_id=?", me.Dataset[i].ID).Find(&me.Dataset[i].ContactPoint) + db.Where("dataset_id=?", me.Dataset[i].ID).Find(&me.Dataset[i].Distributions) + } +} + +//Delete deletes the catalog and its hierarcy +func (me *Catalog) Delete(db *gorm.DB) { + datasets := []int64{} + db.Model(&Dataset{}).Where(&Dataset{}, "CatalogID = ?", me.ID).Pluck("ID", &datasets) + for _, d := range datasets { + db.Delete(&Publisher{}).Where("DatasetID=?", d) + db.Delete(&Distribution{}).Where("DatasetID=?", d) + db.Delete(&ContactPoint{}).Where("DatasetID=?", d) + } + db.Delete(&Dataset{}).Where("CatalogID=?", me.ID) + db.Delete(me) +} + +//GetDatasets retreives all datasets +func (me *Catalog) GetDatasets(db *gorm.DB) { + db.Table("datasets").Where("catalog_id = ?", me.ID).Find(&me.Dataset) + for i := 0; i < len(me.Dataset); i++ { + db.Table("contact_points").Where("dataset_id = ?", me.Dataset[i].ID).Find(&me.Dataset[i].ContactPoint) + db.Table("publishers").Where("dataset_id = ?", me.Dataset[i].ID).Find(&me.Dataset[i].Publisher) + db.Table("distributions").Where("dataset_id = ?", me.Dataset[i].ID).Find(&me.Dataset[i].Distributions) + } +} diff --git a/store/contactpoint.go b/store/contactpoint.go new file mode 100644 index 0000000..df42667 --- /dev/null +++ b/store/contactpoint.go @@ -0,0 +1,12 @@ +package store + +import "github.com/jinzhu/gorm" + +// ContactPoint will store all the information on the contactPoint field. +type ContactPoint struct { + gorm.Model + DatasetID uint `json:"-"` + MetaDataType string `json:"type,omitempty"` + Fn string `json:"fn,omitempty"` + HasEmail string `json:"hasEmail,omitempty"` +} diff --git a/store/dataset.go b/store/dataset.go new file mode 100644 index 0000000..a8c74ff --- /dev/null +++ b/store/dataset.go @@ -0,0 +1,75 @@ +package store + +import ( + "bytes" + "encoding/json" + "strings" + + "github.com/buger/jsonparser" + "github.com/jinzhu/gorm" +) + +// Dataset stores each entry in dataset field of catalog. +type Dataset struct { + gorm.Model + CatalogID uint `json:"-"` + MetadataType string `json:"type,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Modified string `json:"modified,omitempty"` + AccessLevel string `json:"accessLevel,omitempty"` + Identifier string `json:"identifier,omitempty"` + License string `json:"license,omitempty"` + Publisher Publisher `json:"publisher,omitempty"` + ContactPoint ContactPoint `json:"contactPoint,omitempty"` + Distributions []Distribution `json:"distribution,omitempty"` + Keywords string `json:"keyword"` + BureauCodes string `json:"bureauCode"` + ProgramCodes string `json:"programCode"` +} + +//Parse parse dataset +//Utility function +func (me *Dataset) Parse(data []byte) error { + var err error + me.MetadataType, err = jsonparser.GetString(data, "@type") + if err != nil { + return err + } + me.Title, _ = jsonparser.GetString(data, "title") + me.Description, _ = jsonparser.GetString(data, "description") + me.Modified, _ = jsonparser.GetString(data, "modified") + me.AccessLevel, _ = jsonparser.GetString(data, "accessLevel") + me.Identifier, _ = jsonparser.GetString(data, "identifier") + me.License, _ = jsonparser.GetString(data, "license") + + //publisher + me.Publisher = Publisher{} + me.Publisher.MetaDataType, _ = jsonparser.GetString(data, "publisher", "@type") + me.Publisher.Name, _ = jsonparser.GetString(data, "publisher", "name") + + //contact point + me.ContactPoint = ContactPoint{} + me.ContactPoint.MetaDataType, _ = jsonparser.GetString(data, "contactPoint", "@type") + me.ContactPoint.Fn, _ = jsonparser.GetString(data, "contactPoint", "fn") + me.ContactPoint.HasEmail, _ = jsonparser.GetString(data, "contactPoint", "hasEmail") + + me.Distributions = []Distribution{} + if distributionsData, _, _, err := jsonparser.Get(data, "distribution"); err == nil { + json.NewDecoder(bytes.NewBuffer(distributionsData)).Decode(&me.Distributions) + } + + var strArray []string + keywordsData, _, _, _ := jsonparser.Get(data, "keyword") + json.NewDecoder(bytes.NewBuffer(keywordsData)).Decode(&strArray) + me.Keywords = strings.Join(strArray, ",") + + bureauCodeData, _, _, _ := jsonparser.Get(data, "bureauCode") + json.NewDecoder(bytes.NewBuffer(bureauCodeData)).Decode(&strArray) + me.BureauCodes = strings.Join(strArray, ",") + + programCodesData, _, _, _ := jsonparser.Get(data, "programCode") + json.NewDecoder(bytes.NewBuffer(programCodesData)).Decode(&strArray) + me.ProgramCodes = strings.Join(strArray, ",") + return err +} diff --git a/store/distribution.go b/store/distribution.go new file mode 100644 index 0000000..57bc348 --- /dev/null +++ b/store/distribution.go @@ -0,0 +1,19 @@ +package store + +import "github.com/jinzhu/gorm" + +// Distribution will store all the information on the distribution field. +type Distribution struct { + gorm.Model + DatasetID uint `json:"-"` + MetaDataType string `json:"type,omitempty"` + DownloadURL string `json:"downloadURL,omitempty"` + AccessURL string `json:"accessURL,omitempty"` + MediaType string `json:"mediaType,omitempty"` + Format string `json:"format,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + DescribedBy string `json:"describedBy,omitempty"` + DescribedByType string `json:"describedByType,omitempty"` + ConformsTo string `json:"conformsTo,omitempty"` +} diff --git a/store/publisher.go b/store/publisher.go new file mode 100644 index 0000000..7d97429 --- /dev/null +++ b/store/publisher.go @@ -0,0 +1,11 @@ +package store + +import "github.com/jinzhu/gorm" + +// Publisher will store all the information on the publisher field. +type Publisher struct { + gorm.Model + DatasetID uint `json:"-"` + MetaDataType string `json:"@type,omitempty"` + Name string `json:"name,omitempty"` +} diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..ceff030 --- /dev/null +++ b/store/store.go @@ -0,0 +1,17 @@ +package store + +import "github.com/jinzhu/gorm" + +//OpenDBconnection To open & setup db connection +func OpenDBconnection() (*gorm.DB, error) { + db, err := gorm.Open("sqlite3", "velocityworks.db") + if err != nil { + return nil, err + } + err = db.AutoMigrate(&Catalog{}, &Distribution{}, &Publisher{}, + &ContactPoint{}, &Dataset{}).Error + if err != nil { + return nil, err + } + return db, nil +} diff --git a/utils/template.go b/utils/template.go new file mode 100644 index 0000000..70ca46b --- /dev/null +++ b/utils/template.go @@ -0,0 +1,19 @@ +package utils + +import ( + "html/template" + "io" + + "github.com/labstack/echo" +) + +// Template implemented the Renderer interface from echo. +type Template struct { + Template *template.Template +} + +// Render implements Renderer interface. +func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { + c.Logger().Infof("inside render :%#v", data) + return t.Template.ExecuteTemplate(w, name, data) +}