initial commit

Author: claygregory

.eslintrc.yml

@@ -0,0 +1,21 @@
+
+env:
+  es6: true
+  node: true
+extends: 'eslint:recommended'
+parserOptions:
+  sourceType: module
+rules:
+  indent:
+    - error
+    - 2
+  linebreak-style:
+    - error
+    - unix
+  quotes:
+    - error
+    - single
+  semi:
+    - error
+    - always
+  no-console: 0

.gitignore

@@ -0,0 +1 @@
+node_modules

LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Clay Gregory
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,115 @@
+
+# CloudFront Access Log Parser
+
+This is a log parser for Cloudfront Web Distribution and RTMP Distribution access logs. It can be used for directly or with the Node.js Stream API.
+
+# Installation
+
+```bash
+npm install --save cloudfront-log-parser
+```
+
+## Usage Examples
+
+### Synchronous API
+Given a string or Buffer of a log file, the `parse` function can be called directly, returning an array of parsed log entries.
+```javascript
+const CloudFrontParser = require('cloudfront-log-parser');
+const accesses = CloudFrontParser.parse(<Buffer or string of log file>, { format: 'web' });
+//accesses = array of objects, see below for format
+```
+
+### Callback API
+If `parse` is provided with a callback function, it will be called with an array of parsed entries as the result.
+```javascript
+const CloudFrontParser = require('cloudfront-log-parser');
+CloudFrontParser.parse(<Buffer or string of log file>, { format: 'web' }, function (err, accesses) {
+  //accesses = array of objects, see below for format
+});
+```
+
+### Node.js Stream API
+
+The parser also implements `stream.Transform` for use in Node.js Streams.
+
+```javascript
+const CloudFrontParser = require('cloudfront-log-parser');
+const fs = require('fs');
+const zlib = require('zlib');
+
+const parser = new CloudFrontParser({ format: 'web' });
+parser.on('readable', function(){
+  let access;
+  while(access = parser.read()){
+    //access = parsed entry object
+  }
+});
+
+fs.createReadStream('./somelogfile.gz')
+  .pipe(zlib.createGunzip())
+  .pipe(parser);
+```
+
+### Options
+
+Only two configuration options are currently in effect: format and version. The parser defaults to `web` to handle the web distribution file format. If logs are from an RTMP distribution, this value should be set to `rtmp`. Currently all CloudFront logs are on version 1.0; should future versions appear, the `version` option will serve as an override.
+
+```javascript
+const options = {
+  format: 'web|rtmp',
+  version: '1.0'
+};
+```
+
+## Result Object
+
+### Web Distribution Format
+
+```javascript
+{ 'date': '2017-02-09',
+  'time': '17:50:17',
+  'x-edge-location': 'MUC51',
+  'sc-bytes': '2797',
+  'c-ip': '192.168.0.123',
+  'cs-method': 'GET',
+  'cs-host': 'yourdistribution.cloudfront.net',
+  'cs-uri-stem': '/',
+  'sc-status': '200',
+  'cs-referer': '-',
+  'cs-user-agent': 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
+  'cs-uri-query': '-',
+  'cs-cookie': '-',
+  'x-edge-result-type': 'Hit',
+  'x-edge-request-id': 'sjXpb8nMq_1ewovZ6nrojpvxIETPbo7EhF2RNtPZ_zfd0MtOW6pjlg==',
+  'x-host-header': 'example.com',
+  'cs-protocol': 'https',
+  'cs-bytes': '148',
+  'time-taken': '0.002',
+  'x-forwarded-for': '-',
+  'ssl-protocol': 'TLSv1.2',
+  'ssl-cipher': 'ECDHE-RSA-AES128-GCM-SHA256',
+  'x-edge-response-result-type': 'Hit',
+  'cs-protocol-version': 'HTTP/1.1' }
+```
+
+### RTMP Distribution Format
+
+```javascript
+{ 'date': '2010-03-12',
+  'time': '23:56:21',
+  'x-edge-location': 'SEA4',
+  'c-ip': '192.0.2.199',
+  'x-event': 'stop',
+  'sc-bytes': '429822014',
+  'x-cf-status': 'OK',
+  'x-cf-client-id': 'bfd8a98bed0840d2b871b7f6adf9908f',
+  'cs-uri-stem': 'rtmp://yourdistribution.cloudfront.net/cfx/st',
+  'cs-uri-query': 'key=value',
+  'c-referrer': 'http://player.example.com/player.swf',
+  'x-page-url': 'http://www.example.com/video',
+  'c-user-agent': 'LNX 10,0,32,18' }
+```
+
+##License
+
+See the included [LICENSE](LICENSE.md) for rights and limitations under the terms of the MIT license.

index.js

@@ -0,0 +1,124 @@
+'use strict';
+
+const stream = require('stream');
+const util = require('util');
+
+const formats = {
+  'web_v1.0': [
+    'date', 'time', 'x-edge-location', 'sc-bytes', 'c-ip',
+    'cs-method', 'cs-host', 'cs-uri-stem', 'sc-status',
+    'cs-referer', 'cs-user-agent', 'cs-uri-query', 'cs-cookie',
+    'x-edge-result-type', 'x-edge-request-id', 'x-host-header',
+    'cs-protocol', 'cs-bytes', 'time-taken', 'x-forwarded-for',
+    'ssl-protocol', 'ssl-cipher', 'x-edge-response-result-type',
+    'cs-protocol-version'
+  ],
+
+  'rtmp_v1.0': [
+    'date', 'time', 'x-edge-location', 'c-ip', 'x-event',
+    'sc-bytes', 'x-cf-status', 'x-cf-client-id', 'cs-uri-stem',
+    'cs-uri-query', 'c-referrer', 'x-page-url', 'c-user-agent'
+  ]
+};
+
+const option_defaults = {
+  format: 'web',
+  version: '1.0'
+};
+
+const decode_field = val => {
+  return decodeURIComponent(
+    val.replace(/%2522/g, '"')
+      .replace(/%255C/g, '\\')
+      .replace(/%2520/g, ' ')
+  );
+};
+
+const CloudFrontTransform = function (options) {
+
+  this.options = options || {};
+  stream.Transform.call(this, { objectMode: true });
+};
+util.inherits(CloudFrontTransform, stream.Transform);
+
+CloudFrontTransform.prototype._transform = function (chunk, encoding, done) {
+
+  encoding = encoding || 'utf8';
+  if (Buffer.isBuffer(chunk)) {
+    if (encoding == 'buffer') chunk = chunk.toString();
+    else chunk = chunk.toString(encoding);
+  }
+
+  if (this._lastLineBuffer) chunk = this._lastLineBuffer + chunk;
+
+  const lines = chunk.split('\n');
+  this._lastLineBuffer = lines.splice(lines.length - 1, 1)[0];
+
+  parse(lines, this.options).forEach(this.push.bind(this));
+  done();
+};
+
+CloudFrontTransform.prototype._flush = function (done) {
+
+  if (this._lastLineBuffer) {
+    parse(this._lastLineBuffer, this.options).forEach(this.push.bind(this));
+    this._lastLineBuffer = null;
+  }
+
+  done();
+};
+
+function parse(data, options, callback) {
+
+  let parsed;
+  let err;
+  try {
+
+    if (Buffer.isBuffer(data)) data = data.toString();
+
+    if (typeof data === 'string') data = data.split('\n');
+
+    parsed = data
+      .filter(line => !line.startsWith('#'))
+      .filter(line => line.length > 0)
+      .map(line => parseLine(line, options));
+
+  } catch (e) {
+    err = e;
+    if (!callback) throw e; 
+  }
+
+  if (callback) callback(err, parsed);
+  else return parsed;
+}
+
+function parseLine(line, options) {
+
+  options = options || {};
+
+  let format = options.format;
+  if (format === undefined) format = option_defaults.format;
+
+  let version = options.version;
+  if (version === undefined) version = option_defaults.version;
+
+  const headings = formats[`${format}_v${version}`];
+  if (!headings) throw new Error(`Format not recognized: ${format}`);
+
+  const line_arr = line.split('\t');
+  return _zipLine(line_arr, headings);
+}
+
+function _zipLine(arr, headings) {
+  const result = {};
+  for (let i = 0; i < Math.min(arr.length, headings.length); i++) {
+    const field = headings[i];
+    result[field] = decode_field(arr[i]);
+  }
+
+  return result;
+}
+
+module.exports = CloudFrontTransform;
+module.exports.parse = parse;
+module.exports.parseLine = parseLine;

package.json

@@ -0,0 +1,15 @@
+{
+  "name": "cloudfront-log-parser",
+  "version": "0.0.1",
+  "description": "Parser for CloudFront access logs",
+  "author": "Clay Gregory <[email protected]> (https://claygregory.com/)",
+  "repository": "claygregory/node-cloudfront-log-parser",
+  "homepage": "https://github.com/claygregory/node-cloudfront-log-parser",
+  "license": "MIT",
+  "main": "index.js",
+  "dependencies": {
+  },
+  "devDependencies": {
+    "eslint": "^3.13.1"
+  }
+}