Node.js streaming csv downloads proxy

Streaming downloads proxy service with Node.js ismael celis @ismasan

bootic.net - Hosted e-commerce in South America

background job Email attachment Previous setup

Previous setup • Memory limitations • Email deliverability • Code bloat • inﬂexible

New setup • Monolithic micro • Leverage existing API

New setup • Monolithic micro • Leverage existing API curl -H “Authorization: Bearer xxx” https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2

API -> CSV Stream // pipe generated CSV onto the HTTP response  var writer = csv.createCsvStreamWriter(response)    // Turn a series of paginated requests   // to the backend API into a stream of data  var stream = apistream.instance(uri, token)    // Pipe data stream into CSV writer  stream.pipe(writer)

API -> CSV Stream response.setHeader('Content-Type', ‘text/csv');   response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');

API -> mappers -> CSV Stream {  "code": "123EFCD",  "total": 80000,  "status": "shipped",  "date": "2014-02-03",  "items": [  {"product_title": "iPhone 5", "units": 2, "unit_price": 30000},  {"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000}  ]  } code, total, date, status, product, units, unit_price, total  2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 80000  3 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000

API -> mappers -> CSV Stream var OrderMapper = csvmapper.define(function () {    this.scope('items', function () {  this  .map('id', '/id')  .map('order', '/code')  .map('status', '/status')  .map('discount', '/discount_total')  .map('shipping price', '/shipping_total')  .map('total', '/total')  .map('year', '/updated_on', year)  .map('month', '/updated_on', month)  .map('day', '/updated_on', day)  .map('payment method', '/payment_method_type')  .map('name', '/contact/name')  .map('email', '/contact/email')  .map('address', '/address', address)  .map('product', 'product_title')  .map('variant', 'variant_title')  .map('sku', 'variant_sku')  .map('unit price', 'unit_price')

API -> mappers -> CSV Stream var writer = csv.createCsvStreamWriter(res);    var stream = apistream.instance(uri, token)    var mapper = new OrdersMapper()    // First line in CSV is the headers  writer.writeRecord(mapper.headers())    // mapper.eachRow() turns a single API resource into 1 or more CSV rows  stream.on('item', function (item) {  mapper.eachRow(item, function (row) {  writer.writeRecord(row)  })  })

API -> mappers -> CSV Stream stream.on('item', function (item) {  mapper.eachRow(item, function (row) {  writer.writeRecord(row)  })  })

Paremeter deﬁnitions c.net/v1/orders.json? created_at:gte=2014-02-01 & page=2

Paremeter deﬁnitions var OrdersParams = params.define(function () {  this  .param('sort', 'updated_on:desc')  .param('per_page', 20)  .param('status', 'closed,pending,invalid,shipped')  })

Paremeter deﬁnitions var params = new OrdersParams(request.query)    // Compose API url using sanitized / defaulted params  var uri = "https://api.com/orders?" + params.query;    var stream = apistream.instance(uri, token)

JSON Web Tokens headers . claims . signature http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk

JSON Web Tokens headers {“typ":"JWT", "alg":"HS256"} claims {  “shop_id":"acme",  "iat":1300819380,  "aud":"orders",  "filters": {"status": "shipped"}  } signature + Base64 + Base64 HMAC SHA-256 (headers + claims, secret) + Base64

Rails: Generate token (Ruby) # controllers/downloads_controller.rb  def create   url = Rails.application.config.downloads_host  claims = params[:download_options]  # Add an issued_at timestamp  claims[:iat] = (Time.now.getutc.to_f * 1000).to_i  # Scope data on current account  claims[“shop_id"] = current_shop.id  # generate JWT  token = JWT.encode(claims, Rails.application.config.downloads_secret)    # Redirect to download URL. Browser will trigger download dialog  redirect_to “#{url}?jwt=#{token}"   end

Rails: Generate token (Ruby) claims[:iat] = (Time.now.getutc.to_f * 1000).to_i    claims[“shop_id"] = current_shop.id    token = JWT.encode(claims, secret)    redirect_to "#{url}?jwt=#{token}"

Node: validate JWT var TTL = 60000;    var tokenMiddleware = function(req, res, next){  try{  var decoded = jwt.decode(req.query.jwt, secret);    if(decoded.shop_id != req.param(‘shop_id') {  res.send(400, ‘JWT and query shop ids do not match');  return  }    var now = new Date(),  utc = getUtcCurrentDate();    if(utc - Number(decoded.iat) > TTL) {  res.send(401, "Web token has expired")  return  }    req.query = decoded  // all good, carry on  next()    } catch(e) {  res.send(401, 'Unauthorized or invalid web token');  }  }

Node: validate JWT var decoded = jwt.decode(req.query.jwt, secret); ?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU

Node: validate JWT if(decoded.shop_id != req.param(‘shop_id') {  res.send(400, ‘JWT and query shop ids do not match');  return  }

Node: validate JWT var now = new Date(),  utc = getUtcCurrentDate();    if(utc - Number(decoded.iat) > TTL) {  res.send(401, "Web token has expired")  return  }

Node: validate JWT req.query = decoded   // all good, carry on  next()

Node: HTTP handlers app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...)); app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...)); app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));

} goo.gl/nolmRK ismael celis @ismasan

Node.js streaming csv downloads proxy

More Related Content

What's hot

Recently uploaded

Node.js streaming csv downloads proxy