analyseTemplate.Rmd

<!-- Make sure that the knitr package is installed and loaded. -->
<!-- For more info on the package options see http://yihui.name/knitr/options -->

<!-- Replace below with the title of your project -->
# Summary
Analysed on `r format(Sys.time(), RENDER_DATE_FORMAT)`

Source files : `r FILE_NAMES`

Number of lines : `r nrow(access_log)`

Measured from `r format(min(access_log$ts), RENDER_DATE_FORMAT)` 
to `r format(max(access_log$ts), RENDER_DATE_FORMAT)`

Regex used to define requests' categories :
```{r echo=FALSE}
values=unlist(CATEGORIES,use.names = FALSE)
df=data.frame(names=names(CATEGORIES), regexp=sapply(values, cleanStr, USE.NAMES = FALSE))
pander(df, style = 'rmarkdown', split.table = 24000)
``` 

```{r echo=FALSE, results='asis',}
if (max(access_log$response.time_millis) == 0) {
  cat('<span style="color:red">The accesslog file doesn\'t contain duration data. Please disregard of duration reports. You should add "%D" at the end of the accesslog format to enable duration analysis.</span>\n')
}
``` 

# Requests throughput over time
## All responses
<!-- all_request_per_hours.png -->
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
  log("Creating all_request_per_hours")
  g = ggplot(access_log, aes(x = ts)) + 
  geom_density(stat = "bin", binwidth = INTERVAL_IN_SECONDS,
          colour = "black", fill = "darkgreen") + ylab(paste0("Requests/",INTERVAL_AS_TEXT)) + xlab("Time") 
  print(g)
```

## Throughput by category
<!-- all_request_per_hours_by_type.png -->
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating all_request_per_hours_by_type")
g = ggplot(access_log, aes(x = ts)) + 
  geom_density(stat = "bin", binwidth = INTERVAL_IN_SECONDS, position="stack", aes(fill = category, color=category, order=-as.numeric(category))) + 
     ylab(paste0("Requests/",INTERVAL_AS_TEXT)) + xlab("Time") 
print(g)
```

# Response analysis

## Top 10 of most called URL
```{r echo=FALSE}
log("Creating top  most call URL")
tmp = data.table(access_log)[, length(status), by = request]
setkey( tmp, V1)
setnames(tmp, "V1", "Number of requests")
tmp=tail(tmp)
pander(tmp[nrow(tmp):1,], style = 'rmarkdown', split.table = 1000)
```

## Method
<!-- http_method -->
```{r echo=FALSE}
log("Creating http_method distribution")
tmp = data.table(access_log)[, length(ip), by = method]
setkey( tmp, method)
setnames(tmp, "V1", "Number of requests")
tmp$percentage = paste(round(tmp$'Number of requests' / nrow(access_log) * 100,1), "%")
tmp=tmp[order(tmp$'Number of requests', decreasing = TRUE)]
pander(tmp, style = 'rmarkdown', split.table = 1000)
```

```{r echo=FALSE, h=S_HEIGHT, w=S_WIDTH}
log("Creating http_method")
df <- as.data.frame(table(access_log$method))
colnames(df) <- c('method','freq')
g = ggplot(df, aes(x = "", y = freq, fill = method, color = method)) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0) +
  labs(title = "HTTP Methods distribution") + xlab("")
print(g)
```

## HTTP Code
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating response code table")
tmp = data.table(access_log)[, length(ip), by = status]
setkey( tmp, status)
setnames(tmp, "V1", "Number of requests")
tmp$ratio = paste(round(tmp$'Number of requests' / nrow(access_log) * 100,1), "%")
pander(tmp, style = 'rmarkdown', split.table = 1000)
```

<!-- response_code -->
```{r echo=FALSE, h=S_HEIGHT, w=S_WIDTH}
  log("Creating response_code")
  df <- as.data.frame(table(access_log$status))
  colnames(df) <- c('status','freq')
  g = ggplot(df, aes(x = "", y = freq, fill = status, color = status)) +
    geom_bar(width = 1, stat = "identity") +
    coord_polar("y", start = 0) +
    labs(title = "HTTP return code distribution") + xlab("")
  print(g)
```

## Errors
### Errors distribution
<!-- response_error -->
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating response_error")
server.errors <- grep(ERROR_PATTERN,access_log$status)
g = ggplot(access_log[server.errors,], aes(x=status)) + geom_bar(colour="black", fill="red") +
  labs(title = "HTTP errors distribution")
print(g)
```

### Errors over time
<!-- response_error_by_time -->
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("response_error_by_time")
g = ggplot(access_log[server.errors,], aes(x=ts)) +
  geom_density(stat='bin',binwidth=INTERVAL_IN_SECONDS, position="stack") +
  aes(fill = status, color=status, order=-as.numeric(category)) +
  ylab(paste0('Errors/', INTERVAL_AS_TEXT)) + xlab('Time')
  print(g)
```

# Connected IP
## Info
Number of IPs : `r length(unique(access_log$ip))`

Number of requests per IP :
```{r echo=FALSE}
log("Creating Number of requests per IP")
tmp = data.table(access_log)[, length(status), by = ip]
setkey( tmp, V1)
setnames(tmp, "V1", "Number of requests")
tmpSum = summary(tmp$"Number of requests")
displ = data.frame(matrix(NA,ncol=length(names(tmpSum)),nrow=1))
names(displ)=names(tmpSum)
displ[1,]=tmpSum
pander(displ, style = 'rmarkdown', split.table = 1000)
```

Distribution of the number of requests per IP :
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
setnames(tmp, "Number of requests", "Number_of_requests")
g = ggplot(tmp, aes(x = Number_of_requests)) + 
  geom_density(colour = "black", fill = "darkgreen")  + xlab("Number of requests per IP)") 
print(g)
``` 

## IP with the highest number of connections
```{r echo=FALSE}
log("Creating Most connected IP")
mostClient=tmp[nrow(tmp):1,][1:min(10,nrow(tmp))]
pander(mostClient, style = 'rmarkdown', split.table = 1000)
```

# Response time distribution (ms)
<a name="sometext"></a>
## Response time summary by category
```{r echo=FALSE}
log("Creating all_responsetime_distribution")
distrib = by(access_log, access_log$category, function(x) list(nrow(x), round(mean(x$response.time_millis)), round(quantile(x$response.time_millis, c(0, .25, .50,  .75, .95, 1)))))
displ = analyseDistribution(access_log$response.time_millis, distrib, "Number of Requests")
pander(displ, style = 'rmarkdown', split.table = 1000)
```

## All requests
### Top 10 slowest requests (ms)
```{r echo=FALSE}
log("Creating Top 10 slow request")
df=access_log[with(access_log, order(-response.time_millis)),][1:10,c("response.time_millis", "ts", "category", "method", "url_extract", "status")]
df=as.matrix(df)
pander(df, style = 'rmarkdown', split.table = 1000)
```

### Distribution (ms)
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
g = ggplot(access_log, aes(x = response.time_millis)) + 
  geom_density(colour = "black", fill = "darkgreen")  + xlab("Response time (milliseconds)") 
print(g)
``` 

### Distribution per categories (ms)
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating all_responsetime_distribution_by_type")
xmax = quantile(access_log$response.time_millis, c(PERCENTILE_FOR_DISTRIBUTION))
xmin = min(access_log$response.time_millis)
g = ggplot(access_log, aes(x = response.time_millis)) + 
  geom_density(aes(group=category, colour=category))  + 
  xlab(paste0("Response time (max=",max(access_log$response.time_millis),")")) +
  scale_x_continuous(limits=c(xmin, xmax))
  scale_x_continuous(breaks=c(0, 5000, 5000))+
  coord_cartesian(xlim =c(xmin, xmax))
suppressWarnings(print(g))
```

## Top 5 slowest requests and distribution per category (ms)
<!-- Reponse time table -->
```{r echo=FALSE, comment=NA, results='asis', h=B_HEIGHT, w=B_WIDTH}
for (cat in CATEGORY_NAMES) {
cat("### ", cat)
log(paste("Distribution", cat))
subdata = access_log[access_log$category==cat ,]
  df=subdata[with(subdata, order(-response.time_millis)),][1:5,c("response.time_millis", "ts", "method", "url_extract", "status")]
  df = as.matrix(df)
  cat(pander(df, style = 'rmarkdown', split.table = 1000))
  xmax = quantile(subdata$response.time_millis, c(PERCENTILE_FOR_DISTRIBUTION))
  xmin = min(subdata$response.time_millis)
  g = ggplot(subdata, aes(x = response.time_millis)) + 
    geom_density()  + xlab(paste0("Response time (max=",max(subdata$response.time_millis),")")) +
    coord_cartesian(xlim =c(xmin, xmax))
  suppressWarnings(print(g))
  cat("\n")
}
rm(df, subdata)
``` 

# Reponse time over time
## Global (ms)
```{r echo=FALSE, comment=NA, results='asis', h=B_HEIGHT, w=B_WIDTH}
log("Creating response_time_by_time")
g = ggplot(access_log, aes(ts)) + 
  xlab("Date") + ylab("response time (ms)") +
  geom_point(aes(y = response.time_millis), alpha=0.3) + 
  ggtitle("Response time evolution") +
  stat_smooth(data=access_log, aes(x=ts, y=response.time_millis), colour="red",method = "gam", formula = y ~ s(x, bs = "cs"))
print(g)
```

```{r echo=FALSE, comment=NA, results='asis', h=B_HEIGHT, w=B_WIDTH}
log("Creating response_time_by_time_and_category")
g = ggplot(access_log, aes(ts)) + 
  xlab("Date") + ylab("response time (ms)") +
  geom_point(aes(y = response.time_millis, color=category),alpha=0.3) + 
  ggtitle("Response time evolution by type") +
  stat_smooth(data=access_log, aes(x=ts, y=response.time_millis), colour="red",method = "gam", formula = y ~ s(x, bs = "cs"))
print(g)
```

## Per category (ms)
```{r echo=FALSE, comment=NA, results='asis', h=B_HEIGHT, w=B_WIDTH}
for (cat in CATEGORY_NAMES) {
    log(paste0("Creating response_time_by_time_and_", cat))
    cat("### ", cat, "\n")
    subdata = access_log[access_log$category==cat ,]
    g = ggplot(subdata, aes(ts)) + 
      xlab("Date") + ylab("response time (ms)") +
      geom_point(aes(y = response.time_millis), alpha=0.3) + 
      ggtitle(paste0("Response time evolution for ", cat, " and HTTPCode=200")) +
      stat_smooth(data=subdata, aes(x=ts, y=response.time_millis), colour="red",method = "gam", formula = y ~ s(x, bs = "cs"))
    print(g)
    cat("\n")
}
```

## Per Http method (ms)
```{r echo=FALSE, comment=NA, results='asis', h=B_HEIGHT, w=B_WIDTH}
for (meth in unique(access_log$method)) {
    log(paste0("Creating response_time_by_time_and_", meth))
    cat("### ", meth, "\n")
    subdata = access_log[access_log$method==meth ,]
    g = ggplot(subdata, aes(ts)) + 
      xlab("Date") + ylab("response time (ms)") +
      geom_point(aes(y = response.time_millis), alpha=0.3) + 
      ggtitle(paste0("Response time evolution for ", meth, " and HTTPCode=200")) +
      stat_smooth(data=subdata, aes(x=ts, y=response.time_millis), colour="red",method = "gam", formula = y ~ s(x, bs = "cs"))
    print(g)
    cat("\n")
}
```

# Response size analysis
## Top 10 biggest response
```{r echo=FALSE}
accesslog_with_size = access_log[complete.cases(access_log[,"response.size"]),]
startDate = min(accesslog_with_size$ts)
accesslog_with_size$interval = as.numeric(difftime(accesslog_with_size$ts, startDate), units="secs") %/% INTERVAL_IN_SECONDS
accesslog_with_size$top_ip = as.character(accesslog_with_size$ip)
accesslog_with_size[accesslog_with_size$ip %!in% mostClient$ip, "top_ip"] = "Other"
log("Creating top big response size")
df=accesslog_with_size[with(accesslog_with_size, order(-response.size)),][1:10,c("response.size", "ts", "category", "method", "url_extract", "status")]
df=as.matrix(df)
pander(df, style = 'rmarkdown', split.table = 1000)
```

## Response size summary by category
```{r echo=FALSE}
log("Creating all_responsesize_summary_by_category")
distrib = by(accesslog_with_size, accesslog_with_size$category, function(x) list(round(sum(x$response.size)/1024/1024,1), round(mean(x$response.size)), round(quantile(x$response.size, c(0, .25, .50,  .75, .95, 1)))))
displ = analyseDistribution(accesslog_with_size$response.size, distrib, "Total downloaded (MB)")
pander(displ, style = 'rmarkdown', split.table = 1000)
```

## Response size summary by IP
```{r echo=FALSE}
log("Creating all_responsesize_summary_by_ip")
distrib = by(accesslog_with_size, accesslog_with_size$top_ip, function(x) list(round(sum(x$response.size)/1024/1024,1), round(mean(x$response.size)), round(quantile(x$response.size, c(0, .25, .50,  .75, .95, 1)))))
displ = analyseDistribution(accesslog_with_size$response.size, distrib, "Total downloaded (MB)")
pander(displ, style = 'rmarkdown', split.table = 1000)
```

## Bandwidth over time
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating all_bandwith_over_time")
log(startDate)
distrib = by(accesslog_with_size, accesslog_with_size$interval, function(x) {sum(x$response.size)/1024/1024})
dt = data.frame(date=startDate+as.numeric(names(distrib))*INTERVAL_IN_SECONDS, bandwidth=as.vector(distrib))
```

Bandwidth summary (MB by interval of `r INTERVAL_AS_TEXT` )
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
tmpSum = summary(dt$bandwidth)
displ = data.frame(matrix(NA,ncol=length(names(tmpSum)),nrow=1))
names(displ)=names(tmpSum)
displ[1,]=tmpSum
pander(displ, style = 'rmarkdown', split.table = 1000)

g = ggplot(dt, aes(date, ymin=0)) + 
  xlab("Time") + ylab(paste0('bandwith (MB) by ', INTERVAL_AS_TEXT)) +
  geom_point(aes(y = bandwidth)) + 
  ggtitle("Bandwidth evolution") +
  stat_smooth(data=dt, aes(x=date, y=bandwidth), colour="red",method = "gam", formula = y ~ s(x, bs = "cs"))
print(g)
``` 


## Response size distribution
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating all_responsesize_distribution")
g = ggplot(accesslog_with_size, aes(x = response.size)) + 
  geom_density(colour = "black", fill = "darkgreen")  + xlab("Response size (bytes)") 
print(g)
``` 

## Response size distribution per category
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating all_responsesize_distribution_by_type")
xmax = quantile(accesslog_with_size$response.size, c(PERCENTILE_FOR_DISTRIBUTION))
xmin = min(accesslog_with_size$response.size)
g = ggplot(accesslog_with_size, aes(x = response.size)) + 
  geom_density(aes(group=category, colour=category))  + 
  xlab(paste0("Response size (bytes, max=",max(accesslog_with_size$response.size),")")) +
  coord_cartesian(xlim =c(xmin, 100*xmax))
suppressWarnings(print(g))
```

## Response size distribution per IP
```{r echo=FALSE, h=B_HEIGHT, w=B_WIDTH}
log("Creating all_responsesize_distribution_by_type")
xmax = quantile(accesslog_with_size$response.size, c(PERCENTILE_FOR_DISTRIBUTION))
xmin = min(accesslog_with_size$response.size)
g = ggplot(accesslog_with_size, aes(x = response.size)) + 
  geom_density(aes(group=top_ip, colour=top_ip))  + 
  xlab(paste0("Response size (bytes, max=",max(accesslog_with_size$response.size),")")) +
  coord_cartesian(xlim =c(xmin, xmax))
suppressWarnings(print(g))
```

## Top 5 biggest requests and distribution per category
<!-- Reponse time table -->
```{r echo=FALSE, comment=NA, results='asis', h=B_HEIGHT, w=B_WIDTH}
for (cat in CATEGORY_NAMES) {
cat("### ", cat)
log(paste("Distribution", cat))
subdata = accesslog_with_size[accesslog_with_size$category==cat ,]
  df=subdata[with(subdata, order(-response.size)),][1:5,c("response.size", "ts", "method", "url_extract", "status")]
  df = as.matrix(df)
  cat(pander(df, style = 'rmarkdown', split.table = 1000))
  xmax = quantile(subdata$response.size, c(PERCENTILE_FOR_DISTRIBUTION))
  xmin = min(subdata$response.size)
  g = ggplot(subdata, aes(x = response.size)) + 
    geom_density()  + xlab(paste0("Response size (bytes, max=",max(subdata$response.size),")")) +
    coord_cartesian(xlim =c(xmin, xmax))
  suppressWarnings(print(g))
  cat("\n")
}
rm(df, subdata)
```