Merge pull request #112 from subutai-io/dev

Dev

Makefile

@@ -5,7 +5,8 @@ ifeq (${GIT_BRANCH}, )
 	GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD | grep -iv head)
 endif
 ifneq (${GIT_BRANCH}, )
-	VERSION:=${VERSION}-SNAPSHOT
+	#VERSION:=${VERSION}-SNAPSHOT
+	VERSION:=6.0.1-SNAPSHOT
 endif
 COMMIT=$(shell git rev-parse HEAD)
 

db/db.go

@@ -64,13 +64,6 @@ func Write(owner, key, value string, options ...map[string]string) {
 			b.Put([]byte("date"), now)
 			b.Put([]byte("name"), []byte(value))
 
-			// Getting file size
-			if f, err := os.Open(config.Storage.Path + key); err == nil {
-				fi, _ := f.Stat()
-				f.Close()
-				b.Put([]byte("size"), []byte(fmt.Sprint(fi.Size())))
-			}
-
 			// Adding search index for files
 			b, _ = tx.Bucket(search).CreateBucketIfNotExists([]byte(strings.ToLower(value)))
 			b.Put(now, []byte(key))
@@ -90,6 +83,16 @@ func Write(owner, key, value string, options ...map[string]string) {
 								c.Put([]byte(owner), []byte("w"))
 							}
 						}
+					case "md5", "sha256":
+						if c, err := b.CreateBucketIfNotExists([]byte("hash")); err == nil {
+							c.Put([]byte(k), []byte(v))
+							// Getting file size
+							if f, err := os.Open(config.Storage.Path + v); err == nil {
+								fi, _ := f.Stat()
+								f.Close()
+								b.Put([]byte("size"), []byte(fmt.Sprint(fi.Size())))
+							}
+						}
 					case "tags":
 						if c, err := b.CreateBucketIfNotExists([]byte("tags")); err == nil && len(v) > 0 {
 							for _, v := range strings.Split(v, ",") {
@@ -126,7 +129,12 @@ func Delete(owner, repo, key string) (total int) {
 		var filename []byte
 
 		owned := CheckRepo(owner, "", key)
-		total = CheckRepo("", "", key)
+		md5, _ := Hash(key)
+		if key == md5 {
+			total = CheckRepo("", "", key)
+		} else {
+			total = CheckRepo("", "", md5)
+		}
 
 		// Deleting user association with file
 		if b := tx.Bucket(bucket).Bucket([]byte(key)); b != nil {
@@ -153,7 +161,7 @@ func Delete(owner, repo, key string) (total int) {
 		}
 
 		// Removing indexes and file only if no file owners left
-		if total == 1 {
+		if total == 1 || key != md5 {
 			// Deleting search index
 			if b := tx.Bucket(search).Bucket(bytes.ToLower(filename)); b != nil {
 				b.ForEach(func(k, v []byte) error {
@@ -191,18 +199,40 @@ func Read(key string) (name string) {
 	return name
 }
 
-func Info(hash string) map[string]string {
+// Hash returns hash sums by ID
+func Hash(key string) (md5, sha256 string) {
+	db.View(func(tx *bolt.Tx) error {
+		if b := tx.Bucket(bucket).Bucket([]byte(key)); b != nil {
+			if b := b.Bucket([]byte("hash")); b != nil {
+				if value := b.Get([]byte("md5")); value != nil {
+					md5 = string(value)
+				}
+				if value := b.Get([]byte("sha256")); value != nil {
+					sha256 = string(value)
+				}
+			}
+		}
+		return nil
+	})
+	return md5, sha256
+}
+
+func Info(id string) map[string]string {
 	list := make(map[string]string)
 	db.View(func(tx *bolt.Tx) error {
-		if b := tx.Bucket(bucket).Bucket([]byte(hash)); b != nil {
+		if b := tx.Bucket(bucket).Bucket([]byte(id)); b != nil {
 			b.ForEach(func(k, v []byte) error {
 				list[string(k)] = string(v)
 				return nil
 			})
+			if hash := b.Bucket([]byte("hash")); hash != nil {
+				list["md5"] = string(hash.Get([]byte("md5")))
+				list["sha256"] = string(hash.Get([]byte("sha256")))
+			}
 		}
 		return nil
 	})
-	list["id"] = hash
+	list["id"] = id
 	list["owner"] = "subutai"
 	return list
 }
@@ -674,6 +704,16 @@ func CheckRepo(owner, repo, hash string) (val int) {
 		reps = []string{"apt", "template", "raw"}
 	}
 	db.View(func(tx *bolt.Tx) error {
+		if b := tx.Bucket(bucket); b != nil {
+			b.ForEach(func(k, v []byte) error {
+				if b := b.Bucket(k).Bucket([]byte("hash")); b != nil {
+					if string(b.Get([]byte("md5"))) == hash {
+						val++
+					}
+				}
+				return nil
+			})
+		}
 		if b := tx.Bucket(bucket).Bucket([]byte(hash)); b != nil {
 			if c := b.Bucket([]byte("type")); c != nil {
 				for _, v := range reps {

download/download.go

@@ -21,7 +21,7 @@ import (
 type ListItem struct {
 	ID           string            `json:"id"`
 	Hash         hashsums          `json:"hash"`
-	Size         int               `json:"size,omitempty"`
+	Size         int               `json:"size"`
 	Name         string            `json:"name,omitempty"`
 	Tags         []string          `json:"tags,omitempty"`
 	Owner        []string          `json:"owner,omitempty"`
@@ -41,38 +41,30 @@ type hashsums struct {
 
 // Handler provides download functionality for all artifacts.
 func Handler(repo string, w http.ResponseWriter, r *http.Request) {
-	hash := r.URL.Query().Get("hash")
+	id := r.URL.Query().Get("id")
 	name := r.URL.Query().Get("name")
-	if len(r.URL.Query().Get("id")) > 0 {
-		hash = r.URL.Query().Get("id")
-		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
-			hash = tmp[1]
-		}
-	}
-	if len(hash) == 0 && len(name) == 0 {
-		io.WriteString(w, "Please specify hash or name")
+	if len(id) == 0 && len(name) == 0 {
+		io.WriteString(w, "Please specify id or name")
 		return
 	} else if len(name) != 0 {
-		hash = db.LastHash(name, repo)
+		id = db.LastHash(name, repo)
 	}
 
-	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
+	if len(db.Read(id)) > 0 && !db.Public(id) && !db.CheckShare(id, db.CheckToken(r.URL.Query().Get("token"))) {
 		w.WriteHeader(http.StatusNotFound)
 		w.Write([]byte("Not found"))
 		return
 	}
 
-	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
-	// 	torrent.AddTorrent(hash)
-	// 	w.WriteHeader(http.StatusAccepted)
-	// 	w.Write([]byte(torrent.Info(hash)))
-	// 	return
-	// }
+	path := config.Storage.Path + id
+	if md5, sha256 := db.Hash(id); len(md5) != 0 || len(sha256) != 0 {
+		path = config.Storage.Path + md5
+	}
 
-	f, err := os.Open(config.Storage.Path + hash)
+	f, err := os.Open(path)
 	defer f.Close()
 
-	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
+	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+id, err) || len(id) == 0 {
 		if len(config.CDN.Node) > 0 {
 			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
 			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
@@ -103,9 +95,9 @@ func Handler(repo string, w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
 	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
 
-	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
+	if name = db.Read(id); len(name) == 0 && len(config.CDN.Node) > 0 {
 		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
-		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
+		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + id)
 		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
 			var info ListItem
 			rsp, err := ioutil.ReadAll(resp.Body)
@@ -120,7 +112,7 @@ func Handler(repo string, w http.ResponseWriter, r *http.Request) {
 			resp.Body.Close()
 		}
 	} else {
-		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
+		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(id)+"\"")
 	}
 
 	io.Copy(w, f)
@@ -184,7 +176,6 @@ func Info(repo string, r *http.Request) []byte {
 		} else {
 			info = db.Info(k)
 		}
-
 		item := formatItem(info, repo, name)
 
 		if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
@@ -238,7 +229,7 @@ func formatItem(info map[string]string, repo, name string) ListItem {
 
 	item := ListItem{
 		ID:           info["id"],
-		Hash:         hashsums{Md5: info["id"], Sha256: ""},
+		Hash:         hashsums{Md5: info["md5"], Sha256: info["sha256"]},
 		Name:         strings.Split(info["name"], "-subutai-template")[0],
 		Tags:         db.FileField(info["id"], "tags"),
 		Owner:        db.FileField(info["id"], "owner"),
@@ -256,7 +247,9 @@ func formatItem(info map[string]string, repo, name string) ListItem {
 		item.Version = info["Version"]
 		item.Architecture = info["Architecture"]
 	}
-
+	if len(item.Hash.Md5) == 0 {
+		item.Hash.Md5 = item.ID
+	}
 	return item
 }
 

raw/raw.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"strings"
 
+	uuid "github.com/satori/go.uuid"
 	"github.com/subutai-io/agent/log"
 
 	"github.com/subutai-io/gorjun/db"
@@ -18,15 +19,17 @@ func Upload(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		info := map[string]string{
+			"md5":  hash,
 			"type": "raw",
 		}
 		r.ParseMultipartForm(32 << 20)
 		if len(r.MultipartForm.Value["version"]) != 0 {
 			info["version"] = r.MultipartForm.Value["version"][0]
 		}
 		_, header, _ := r.FormFile("file")
-		db.Write(owner, hash, header.Filename, info)
-		w.Write([]byte(hash))
+		id := uuid.NewV4().String()
+		db.Write(owner, id, header.Filename, info)
+		w.Write([]byte(id))
 		log.Info(header.Filename + " saved to raw repo by " + owner)
 	}
 }

template/template.go

@@ -9,6 +9,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/satori/go.uuid"
+
 	"github.com/subutai-io/agent/log"
 
 	"fmt"
@@ -45,8 +47,8 @@ func readTempl(hash string) (configfile string, err error) {
 }
 
 func getConf(hash string, configfile string) (t *download.ListItem) {
-	t = &download.ListItem{ID: hash}
-
+	t = &download.ListItem{ID: uuid.NewV4().String()}
+	t.Hash.Md5 = hash
 	for _, v := range strings.Split(configfile, "\n") {
 		if line := strings.Split(v, "="); len(line) > 1 {
 			line[0] = strings.TrimSpace(line[0])
@@ -95,6 +97,7 @@ func Upload(w http.ResponseWriter, r *http.Request) {
 		db.Write(owner, t.ID, t.Name+"-subutai-template_"+t.Version+"_"+t.Architecture+".tar.gz", map[string]string{
 			"type":        "template",
 			"arch":        t.Architecture,
+			"md5":         hash,
 			"tags":        strings.Join(t.Tags, ","),
 			"parent":      t.Parent,
 			"version":     t.Version,

upload/upload.go

@@ -116,9 +116,9 @@ func Hash(file string, algo ...string) string {
 }
 
 func Delete(w http.ResponseWriter, r *http.Request) string {
-	hash := r.URL.Query().Get("id")
+	id := r.URL.Query().Get("id")
 	token := r.URL.Query().Get("token")
-	if len(hash) == 0 {
+	if len(id) == 0 {
 		w.WriteHeader(http.StatusBadRequest)
 		w.Write([]byte("Empty file id"))
 		log.Warn(r.RemoteAddr + " - empty file id")
@@ -131,9 +131,9 @@ func Delete(w http.ResponseWriter, r *http.Request) string {
 		log.Warn(r.RemoteAddr + " - Failed to authorize using provided token")
 		return ""
 	}
-	info := db.Info(hash)
+	info := db.Info(id)
 	if len(info) == 0 {
-		log.Warn("File not found by hash")
+		log.Warn("File not found by id")
 		w.WriteHeader(http.StatusNotFound)
 		w.Write([]byte("File not found"))
 		return ""
@@ -147,31 +147,31 @@ func Delete(w http.ResponseWriter, r *http.Request) string {
 		return ""
 	}
 
-	if db.CheckRepo(user, repo[3], hash) == 0 {
-		log.Warn("File " + info["name"] + "(" + hash + ") in " + repo[3] + " repo is not owned by " + user + ", rejecting deletion request")
+	if db.CheckRepo(user, repo[3], id) == 0 {
+		log.Warn("File " + info["name"] + "(" + id + ") in " + repo[3] + " repo is not owned by " + user + ", rejecting deletion request")
 		w.WriteHeader(http.StatusNotFound)
 		w.Write([]byte("File " + info["name"] + " not found or it has different owner"))
 		return ""
 	}
-
-	f, err := os.Stat(config.Storage.Path + hash)
+	md5, _ := db.Hash(id)
+	f, err := os.Stat(config.Storage.Path + md5)
 	if !log.Check(log.WarnLevel, "Reading file stats", err) {
 		db.QuotaUsageSet(user, -int(f.Size()))
 		log.Info("User " + user + ", quota usage -" + strconv.Itoa(int(f.Size())))
 	}
 
-	if db.Delete(user, repo[3], hash) == 0 {
-		log.Warn("Removing " + hash + " from disk")
-		// torrent.Delete(hash)
-		if log.Check(log.WarnLevel, "Removing "+info["name"]+"from disk", os.Remove(config.Storage.Path+hash)) {
+	if db.Delete(user, repo[3], id) == 0 {
+		log.Warn("Removing " + id + " from disk")
+		// torrent.Delete(id)
+		if log.Check(log.WarnLevel, "Removing "+info["name"]+"from disk", os.Remove(config.Storage.Path+md5)) {
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte("Failed to remove file"))
 			return ""
 		}
 	}
 
 	log.Info("Removing " + info["name"] + " from " + repo[3] + " repo")
-	return hash
+	return id
 }
 
 func Share(w http.ResponseWriter, r *http.Request) {

warm-cache.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+while [[ $# -gt 1 ]]; do
+
+case $1 in
+    -e|--environment)
+    	ENV="$2"
+    	shift
+    ;;
+    -a|--address)
+    	addr="$2"
+    	shift
+    ;;
+    *)
+	echo "Unknown option $2"
+	exit 1
+    ;;
+esac
+shift 
+done
+
+for type in template raw; do
+	echo $type
+	curl -s -k https://${ENV}cdn.subut.ai:8338/kurjun/rest/$type/info | jq '.[] | .id' | tr -d '"'|
+	while IFS= read -r ID 
+	do
+     		echo "https://$addr:8338/kurjun/rest/$type/get?id=$ID" 
+     		curl -m 3 -k "https://$addr:8338/kurjun/rest/$type/get?id=$ID" -o /dev/null
+	done
+done