Add proto buffer

xfc1939 · Aug 29, 2016 · e65cfea · e65cfea
1 parent 17ece4e
commit e65cfea
Show file tree

Hide file tree

Showing 12 changed files with 5,434 additions and 10 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -116,6 +116,9 @@ INSTALL(FILES
   src/log_define.h
   src/log_util.h
   src/log_api.h
+  src/log_builder.h
+  src/protobuf-c.h
+  src/sls_logstores.pb-c.h
   DESTINATION include/log_c_sdk)
 
 add_subdirectory(sample)
diff --git a/sample/log_post_logs_sample.c b/sample/log_post_logs_sample.c
@@ -12,14 +12,14 @@ void log_post_logs_sample()
     aos_pool_t *p = NULL;
     aos_status_t *s = NULL;
     aos_pool_create(&p, NULL);
-
+    
     cJSON *root = cJSON_CreateObject();
     cJSON_AddStringToObject(root, "__source__", "127.0.0.1");
     cJSON_AddStringToObject(root, "__topic__", "topic");
-    cJSON *tags = cJSON_CreateObject();
-    cJSON_AddItemToObject(root, "__tags__", tags);
-    cJSON_AddStringToObject(tags, "tag1", "tag1");
-    cJSON_AddStringToObject(tags, "tag2", "tag2");
+    //cJSON *tags = cJSON_CreateObject();
+    //cJSON_AddItemToObject(root, "__tags__", tags);
+    //cJSON_AddStringToObject(tags, "tag1", "tag1");
+    //cJSON_AddStringToObject(tags, "tag2", "tag2");
     cJSON *logs = cJSON_CreateArray();
     cJSON_AddItemToObject(root, "__logs__", logs);
     cJSON *log1 = cJSON_CreateObject();
@@ -36,18 +36,57 @@ void log_post_logs_sample()
     if (aos_status_is_ok(s)) {
         printf("post logs succeeded\n");
     } else {
-	    printf("put logs failed\n");
+        printf("%d\n",s->code);
+        printf("%s\n",s->error_code);
+        printf("%s\n",s->error_msg);
+        printf("%s\n",s->req_id);
+        printf("put logs failed\n");
     }
     cJSON_Delete(root);
     aos_pool_destroy(p);
 }
 
+void log_post_logs_sample2(){
+    aos_pool_t *p = NULL;
+    aos_status_t *s = NULL;
+    aos_pool_create(&p, NULL);
+
+    while(1){
+
+        log_group_builder* bder = log_group_create();
+        add_source(bder,"mSource",sizeof("mSource"));
+        add_topic(bder,"mTopic", sizeof("mTopic"));
+        add_log(bder);
+        add_log_key_value(bder, "K", sizeof("K"), "V", sizeof("V"));
+
+
+
+        //log_http_cont* req =  log_get_http_cont(LOG_ENDPOINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET,NULL, PROJECT_NAME, LOGSTORE_NAME, bder);
+
+        //log_clean_http_cont(req);
+
+        s = log_post_logs_from_proto_buf(LOG_ENDPOINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET,NULL, PROJECT_NAME, LOGSTORE_NAME, bder);
+        if (aos_status_is_ok(s)) {
+            //printf("post logs succeeded\n");
+        } else {
+            printf("%d\n",s->code);
+            printf("%s\n",s->error_code);
+            printf("%s\n",s->error_msg);
+            printf("%s\n",s->req_id);
+            printf("put logs failed\n");
+        }
+
+        log_group_destroy(bder);
+
+    }
+}
+
 int main(int argc, char *argv[])
 {
     if (aos_http_io_initialize("linux-x86_64", 0) != AOSE_OK) {
         exit(1);
     }
-    log_post_logs_sample();
+    log_post_logs_sample2();
     aos_http_io_deinitialize();
     return 0;
 }
diff --git a/src/log_api.c b/src/log_api.c
@@ -6,7 +6,183 @@
 #include "log_auth.h"
 #include "log_util.h"
 #include "log_api.h"
-#include "log_define.h"
+
+log_http_cont* log_get_http_cont(const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken,const char* project,const char* logstore,log_group_builder* bder)
+{
+    aos_pool_t* tmp;
+    apr_pool_create(&tmp, NULL);
+    aos_http_request_t *req = (aos_http_request_t*)apr_palloc(tmp, sizeof(aos_http_request_t));
+    req->pool = tmp;
+
+    aos_string_t project_name, logstore_name;
+    aos_table_t *headers = NULL;
+    log_request_options_t *options = NULL;
+    aos_list_t buffer;
+    unsigned char *md5 = NULL;
+    char *buf = NULL;
+    int64_t buf_len;
+    char *b64_value = NULL;
+    aos_buf_t *content = NULL;
+    aos_status_t *s = NULL;
+
+    options = log_request_options_create(tmp);
+    options->config = log_config_create(tmp);
+    aos_str_set(&(options->config->endpoint), endpoint);
+    aos_str_set(&(options->config->access_key_id), accesskeyId);
+    aos_str_set(&(options->config->access_key_secret), accessKey);
+    if(stsToken != NULL)
+    {
+        aos_str_set(&(options->config->sts_token), stsToken);
+    }
+    options->ctl = aos_http_controller_create(tmp, 0);
+    headers = aos_table_make(options->pool, 5);
+    apr_table_set(headers, LOG_API_VERSION, "0.6.0");
+    apr_table_set(headers, LOG_COMPRESS_TYPE, "lz4");
+    apr_table_set(headers, LOG_SIGNATURE_METHOD, "hmac-sha1");
+    apr_table_set(headers, LOG_CONTENT_TYPE, "application/x-protobuf");
+    aos_str_set(&project_name, project);
+    aos_str_set(&logstore_name, logstore);
+
+    aos_list_init(&buffer);
+
+    log_buf* buff = serialize_to_proto_buf(bder);
+    char *body = buff->data;
+    int org_body_size = (int)buff->length;
+    apr_table_set(headers, LOG_BODY_RAW_SIZE, apr_itoa(options->pool, org_body_size));
+    int compress_bound = LZ4_compressBound(org_body_size);
+    char *compress_data = aos_pcalloc(tmp, compress_bound);
+    int compressed_size = LZ4_compress(body, compress_data, org_body_size);
+    content = aos_buf_pack(tmp, compress_data, compressed_size);
+    aos_list_add_tail(&content->node, &buffer);
+
+    //add Content-MD5
+    buf_len = aos_buf_list_len(&buffer);
+    buf = aos_buf_list_content(tmp, &buffer);
+    md5 = aos_md5(tmp, buf, (apr_size_t)buf_len);
+    b64_value = aos_pcalloc(tmp, 50);
+    int loop = 0;
+    for(; loop < 16; ++loop)
+    {
+        unsigned char a = ((*md5)>>4) & 0xF, b = (*md5) & 0xF;
+        b64_value[loop<<1] = a > 9 ? (a - 10 + 'A') : (a + '0');
+        b64_value[(loop<<1)|1] = b > 9 ? (b - 10 + 'A') : (b + '0');
+        ++md5;
+    }
+    b64_value[loop<<1] = '\0';
+    apr_table_set(headers, LOG_CONTENT_MD5, b64_value);
+
+    aos_http_response_t *resp = NULL;
+    aos_table_t *query_params = NULL;
+    headers = aos_table_create_if_null(options, headers, 0);
+
+    query_params = aos_table_create_if_null(options, query_params, 0);
+
+    log_post_logs_request(options, &project_name, &logstore_name, HTTP_POST,
+                          &req, query_params, headers, &resp);
+    log_write_request_body_from_buffer(&buffer, req);
+
+    int res = AOSE_OK;
+    s = aos_status_create(options->pool);
+    res = log_sign_request(req, options->config);
+
+    log_http_cont* cont_out = (log_http_cont*)apr_palloc(req->pool, sizeof(log_http_cont));
+    cont_out->body.data = compress_data;
+    cont_out->body.length = compressed_size;
+    cont_out->root = req->pool;
+    cont_out->headers = req->headers;
+    cont_out->host = req->host;
+    cont_out->proto = req->proto;
+    cont_out->uri = req->uri;
+
+    return cont_out;
+}
+
+void log_clean_http_cont(log_http_cont* cont)
+{
+    apr_pool_destroy(cont->root);
+}
+
+aos_status_t *log_post_logs_from_proto_buf(const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken, const char *project, const char *logstore, log_group_builder* bder)
+{
+    aos_pool_t *p = bder->root;
+    aos_string_t project_name, logstore_name;
+    aos_table_t *headers = NULL;
+    aos_table_t *resp_headers = NULL;
+    log_request_options_t *options = NULL;
+    aos_list_t buffer;
+    unsigned char *md5 = NULL;
+    char *buf = NULL;
+    int64_t buf_len;
+    char *b64_value = NULL;
+    aos_buf_t *content = NULL;
+    aos_status_t *s = NULL;
+
+    options = log_request_options_create(p);
+    options->config = log_config_create(options->pool);
+    aos_str_set(&(options->config->endpoint), endpoint);
+    aos_str_set(&(options->config->access_key_id), accesskeyId);
+    aos_str_set(&(options->config->access_key_secret), accessKey);
+    if(stsToken != NULL)
+    {
+        aos_str_set(&(options->config->sts_token), stsToken);
+    }
+    options->ctl = aos_http_controller_create(options->pool, 0);
+    headers = aos_table_make(p, 5);
+    apr_table_set(headers, LOG_API_VERSION, "0.6.0");
+    apr_table_set(headers, LOG_COMPRESS_TYPE, "lz4");
+    apr_table_set(headers, LOG_SIGNATURE_METHOD, "hmac-sha1");
+    apr_table_set(headers, LOG_CONTENT_TYPE, "application/x-protobuf");
+    aos_str_set(&project_name, project);
+    aos_str_set(&logstore_name, logstore);
+
+    aos_list_init(&buffer);
+
+    log_buf* buff = serialize_to_proto_buf(bder);
+
+    char *body = buff->data;
+
+    if(body == NULL)
+    {
+        s = aos_status_create(options->pool);
+        aos_status_set(s, 400, AOS_CLIENT_ERROR_CODE, "fail to format proto buffer data");
+        return s;
+    }
+    int org_body_size = (int)buff->length;
+    apr_table_set(headers, LOG_BODY_RAW_SIZE, apr_itoa(options->pool, org_body_size));
+    int compress_bound = LZ4_compressBound(org_body_size);
+    char *compress_data = aos_pcalloc(options->pool, compress_bound);
+    int compressed_size = LZ4_compress(body, compress_data, org_body_size);
+    if(compressed_size <= 0)
+    {
+        s = aos_status_create(options->pool);
+        aos_status_set(s, 400, AOS_CLIENT_ERROR_CODE, "fail to compress json data");
+        return s;
+    }
+    content = aos_buf_pack(options->pool, compress_data, compressed_size);
+    aos_list_add_tail(&content->node, &buffer);
+
+    //add Content-MD5
+    buf_len = aos_buf_list_len(&buffer);
+    buf = aos_buf_list_content(options->pool, &buffer);
+    md5 = aos_md5(options->pool, buf, (apr_size_t)buf_len);
+    b64_value = aos_pcalloc(options->pool, 50);
+    int loop = 0;
+    for(; loop < 16; ++loop)
+    {
+        unsigned char a = ((*md5)>>4) & 0xF, b = (*md5) & 0xF;
+        b64_value[loop<<1] = a > 9 ? (a - 10 + 'A') : (a + '0');
+        b64_value[(loop<<1)|1] = b > 9 ? (b - 10 + 'A') : (b + '0');
+        ++md5;
+    }
+    b64_value[loop<<1] = '\0';
+    apr_table_set(headers, LOG_CONTENT_MD5, b64_value);
+
+    s = log_post_logs_from_buffer(options, &project_name, &logstore_name,
+                                  &buffer, headers, &resp_headers);
+    return s;
+
+}
+
 
 aos_status_t *log_post_logs_with_sts_token(aos_pool_t *p, const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken, const char *project, const char *logstore, cJSON *root)
 {

diff --git a/src/log_api.h b/src/log_api.h
@@ -5,10 +5,14 @@
 #include "aos_string.h"
 #include "aos_status.h"
 #include "log_define.h"
+#include "log_builder.h"
+
 LOG_CPP_START
 
 aos_status_t *log_post_logs_with_sts_token(aos_pool_t *p, const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken, const char *project, const char *logstore, cJSON *root);
 aos_status_t *log_post_logs(aos_pool_t *p, const char *endpoint, const char * accesskeyId, const char *accessKey, const char *project, const char *logstore, cJSON *root);
-
+aos_status_t *log_post_logs_from_proto_buf(const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken, const char *project, const char *logstore, log_group_builder* bder);
+log_http_cont* log_get_http_cont(const char *endpoint, const char * accesskeyId, const char *accessKey, const char *stsToken,const char* project,const char* logstore,log_group_builder* bder);
+void log_clean_http_cont(log_http_cont* cont);
 LOG_CPP_END
 #endif
diff --git a/src/log_auth.c b/src/log_auth.c
@@ -263,9 +263,12 @@ int log_sign_request(aos_http_request_t *req,
         aos_get_gmt_str_time(datestr);
         apr_table_set(req->headers, LOG_DATE, datestr);
     }
-
     res = log_get_signed_headers(req->pool, &config->access_key_id, 
                                  &config->access_key_secret, &canon_res, req);
+
+
+
+
     return res;
 }