Generate aws, azure, and gcp instances cpp from csv

data/generate.py

@@ -0,0 +1,109 @@
+import csv
+import os
+
+def read_csv(input_file, columns_to_include, vendor):
+    output_data = []
+
+    with open(input_file, 'r') as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            selected_data = [row[column] for column in columns_to_include]
+            # Name
+            selected_data[0] = "\"" + selected_data[0] + "\""
+            # Memory
+            try:
+                selected_data[1] = float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[1]))
+            except ValueError:
+                selected_data[1] = 0
+            # vCPU
+            try:
+                selected_data[2] = int(float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[2][:4])))
+            except ValueError:
+                selected_data[2] = 0
+            # Network
+            try:
+                selected_data[3] = int(float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[3])))
+            except ValueError:
+                selected_data[3] = 0
+
+            # Specifics
+            if vendor != "Azure":
+                selected_data[3] *= 1000
+            if vendor == "GCP":
+                try:
+                    selected_data[4] = int(float(''.join(c if c.isdigit() or c == '.' else '' for c in selected_data[4])))
+                except ValueError:
+                    selected_data[4] = 0
+                if (selected_data[4] > selected_data[3]):
+                    selected_data[3] = selected_data[4]
+                selected_data.pop()
+
+            # Network default
+            if selected_data[3] == 0:
+                selected_data[3] = 1000
+
+            output_data.append(selected_data)
+    return output_data
+
+def write_to_file(output_data, output_file):
+    with open(output_file, 'a') as f:
+        f.write("{\n")
+        for idx, data in enumerate(output_data):
+            f.write("{ ")
+            for lidx, value in enumerate(data):
+                f.write(str(value))
+                if lidx < len(data) - 1:
+                    f.write(", ")
+                else:
+                    f.write(" }")
+            if idx < len(output_data) - 1:
+                f.write(",\n")
+            else:
+                f.write(" };\n")
+
+def write_header(vendor, output_file):
+    with open(output_file, 'w') as f:
+        f.write("#include \"cloud/" + vendor.lower() + "_instances.hpp\"\n")
+        f.write("//---------------------------------------------------------------------------\n")
+        f.write("// AnyBlob - Universal Cloud Object Storage Library\n")
+        f.write("// Dominik Durner, 2023\n")
+        f.write("//\n")
+        f.write("// This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.\n")
+        f.write("// If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.\n")
+        f.write("// SPDX-License-Identifier: MPL-2.0\n")
+        f.write("//---------------------------------------------------------------------------\n")
+        f.write("namespace anyblob {\n")
+        f.write("namespace cloud {\n")
+        f.write("//---------------------------------------------------------------------------\n")
+        f.write("using namespace std;\n")
+        f.write("//---------------------------------------------------------------------------\n")
+        f.write("vector<" + vendor + "Instance> " + vendor + "Instance::getInstanceDetails()\n")
+        f.write("// Gets a vector of instance type infos\n")
+        f.write("{\n")
+        f.write("    vector<" + vendor + "Instance> instances =\n")
+
+def write_footer(output_file):
+    with open(output_file, 'a') as f:
+        f.write("return instances;\n")
+        f.write("}\n")
+        f.write("//---------------------------------------------------------------------------\n")
+        f.write("}; // namespace cloud\n")
+        f.write("}; // namespace anyblob\n")
+
+def main():
+    vendors = ["AWS", "Azure", "GCP"]
+    columns = {
+        "AWS": ["API Name", "Instance Memory", "vCPUs", "Network Performance"],
+        "Azure": ["Name", "Memory", "vCPUs", "Expected network bandwidth (Mbps)"],
+        "GCP": ["name", "memoryGiB", "vCpus", "bandwidth", "tier1"],
+    }
+    for vendor in vendors:
+        output_data = read_csv(vendor.lower()+".csv", columns.get(vendor), vendor)
+        output_file = "../src/cloud/" + vendor.lower() + '_instances.cpp'
+        write_header(vendor, output_file)
+        write_to_file(output_data, output_file)
+        write_footer(output_file)
+        os.system("clang-format -i " + output_file)
+
+if __name__ == "__main__":
+    main()

data/sources.txt

@@ -0,0 +1,11 @@
+GCP:
+wget https://gcloud-compute.com/machine-types-regions.csv -O gcp.csv
+sed -i -n -e '1p' -e '/us-east1,/p' gcp.csv
+
+AWS:
+Export csv from website
+https://instances.vantage.sh/?region=us-east-2
+
+Azure:
+Export csv from website
+https://azure-instances.info/?region=us-east-2

include/cloud/provider.hpp

@@ -64,8 +64,8 @@ class Provider {
         double memory;
         /// Number of vCPus
         unsigned vcpu;
-        /// The network performance in Gbit
-        std::string network;
+        /// The network performance in Mbit/s
+        unsigned network;
     };
 
     protected:
@@ -112,6 +112,8 @@ class Provider {
     [[nodiscard]] static std::string getETag(std::string_view header);
     /// Get the upload id from the multipart request body
     [[nodiscard]] static std::string getUploadId(std::string_view body);
+    /// Parse a row from csv file
+    [[nodiscard]] static std::vector<std::string> parseCSVRow(std::string_view body);
 
     /// Create a provider (keyId is access email for GCP/Azure)
     [[nodiscard]] static std::unique_ptr<Provider> makeProvider(const std::string& filepath, bool https = true, const std::string& keyId = "", const std::string& keyFile = "", network::TaskedSendReceiver* sendReceiver = nullptr);

src/cloud/aws.cpp

@@ -65,9 +65,9 @@ Provider::Instance AWS::getInstanceDetails(network::TaskedSendReceiver& sendRece
             if (!instance.type.compare(s))
                 return instance;
 
-        return AWSInstance{string(s), 0, 0, ""};
+        return AWSInstance{string(s), 0, 0, 0};
     }
-    return AWSInstance{"aws", 0, 0, ""};
+    return AWSInstance{"aws", 0, 0, 0};
 }
 //---------------------------------------------------------------------------
 string AWS::getInstanceRegion(network::TaskedSendReceiver& sendReceiver)