Recently, @cimm and me deployed a beta version of JSLogger. We encountered some unexpected issues on the IT (dark) side that delayed our launch. Load balancers and SSL support can be a pain in the as* if you don’t deal with it on a daily basis.
The challenge
JSLogger was built on ExpressJS, a wonderful NodeJS framework. Mongoose wraps the database calls to MongoDB where the auth user data and the log entries are stored.
We used to store the app session data in a RedisStore but we recently switched to MongoStore as it makes life much easier when struggling with one type of database only.
Our app Node server looks like this:
/**
* JSLogger manager
*
* app.js
*/
var express = require('express');
mongooseAuth = require('mongoose-auth');
var everyauth = require('./node_modules/mongoose-auth/node_modules/everyauth');
everyauth.debug = process.env.NODE_ENV === 'production' ? false : true;
var MongoStore = require('connect-mongo')(express);
var mongoAuthDbName = 'jslogger_auth_' + (process.env.NODE_ENV || "development");
var gzippo = require('gzippo');
require('./models/db_connect');
// TODO Figure out how to include the mongooseAuth.middleware() without instatiating a User before
User = require('./models/user');
var oneYear = 31557600000;
var gzippoOptions = process.env.NODE_ENV === 'production' ? {clientMaxAge: oneYear, maxAge: oneYear} : {contentTypeMatch: /none/};
var connectAssetsOptions = process.env.NODE_ENV === 'production' ? {src: __dirname + '/public', minifyBuilds: true} : {src: __dirname + '/public'};
var staticRenderer = process.env.NODE_ENV === 'production' ? gzippo.staticGzip(__dirname + '/public', gzippoOptions) : express.static(__dirname + '/public');
var app = module.exports = express.createServer(
express.bodyParser(),
express.methodOverride(),
express.cookieParser(),
express.session({secret: 'jsloggersecretkey83', store: new MongoStore({db: mongoAuthDbName})}),
require('stylus').middleware({src: __dirname + '/public'}),
staticRenderer,
require('connect-assets')(connectAssetsOptions),
mongooseAuth.middleware()
);
// Configuration
app.configure(function(){
app.set('views', __dirname + '/views');
app.set('view engine', 'jade');
});
app.configure('development', function(){
app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});
app.configure('production', function(){
app.use(express.errorHandler());
});
// Routes
require('./routes/auth')(app);
require('./routes/site')(app);
require('./routes/manage')(app);
mongooseAuth.helpExpress(app);
app.listen(process.env.PORT || 5000);
console.log("JSLogger web manager listening on port %d in %s mode", app.address().port, app.settings.env);
The code above starts JSLogger Manager on port 5000, the part of the app that is responsible for delivering static content.
Two more Node servers run on port 6987 and 6988 that are responsible for logging the entries. One logger handles the HTTP requests:
/**
* JSLogger engine
*
* logger.js
*/
var express = require('express');
require('./models/db_connect');
var app = module.exports = express.createServer();
// Configuration
app.configure(function(){
app.use(express.bodyParser());
app.use(express.methodOverride());
});
app.configure('development', function(){
app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});
app.configure('production', function(){
app.use(express.errorHandler());
});
// Routes
require('./routes/logger')(app);
app.listen(process.env.PORT || 6987);
console.log("JSLogger engine listening on port %d in %s mode", app.address().port, app.settings.env);
And the other logger that runs on port 6988 handles the HTTPS requests, dealing with the SSL connections:
/**
* JSLogger engine on SSL
*
* logger_ssl.js
*/
var express = require('express'),
fs = require('fs');
require('./models/db_connect');
var ca = fs.readFileSync('cert/sub.class1.server.ca.pem').toString();
var privateKey = fs.readFileSync('cert/jslogger.com.key').toString();
var certificate = fs.readFileSync('cert/ssl.pem').toString();
var app = module.exports = express.createServer({
ca : ca,
key: privateKey,
cert: certificate
});
// Configuration
app.configure(function(){
app.use(express.bodyParser());
app.use(express.methodOverride());
});
app.configure('development', function(){
app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});
app.configure('production', function(){
app.use(express.errorHandler());
});
// Routes
require('./routes/logger')(app);
app.listen(process.env.PORT || 6988);
console.log("JSLogger engine listening on port %d in %s mode", app.address().port, app.settings.env);
Creating an SSL Node server with ExpressJS is really easy as you can see in the example above.
So, three Node servers had to run on one dedicated server with MongoDB and SSL support.
Choosing a server
First, we tried to host the project on a Mediatemple dedicated server we already owned and had other sites already running on. It seemed to be great in the beginning, but after the project started to grow, the Node server was reaching the memory limit of 512MB in no time, especially when building and compiling the assets in production mode.
As JSLogger was consuming the entire server memory, slowing down the other sites running on that server as well, we had to move to another solution. More than that, The DV server on Mediatemple runs CentOS by default, a Linux distribution that we are not confortable with. Apache is there by default as well, I needed to get rid of it.
I’ve ended up launching an Amazon EC2 Micro instance with an Ubuntu Server 12 LTS on it for JSLogger. An EC2 Micro instance seems to be enough for JSLogger.
Installing the tools
It took me a couple of minutes to generate a script and install all the tools needed for my project on Ubuntu (kudos to Ruslan Khissamov): http://apptob.org/.
Our three Node servers were running smoothly on port 5000, 6987 and 6988.
Keeping my Node app alive with Upstart and Monit
There are a couple of good tools to monitor your Node app and keep it running but I’m used to Upstart and Monit as I worked with them before in production. There is a really great tutorial about how to install and configure them by Tim Smart.
The tricky part is to configure your Upstart and Monit configs properly. JSLogger Upstart configs look like this:
#!upstart
# /etc/init/jslogger_manager.conf
description "jslogger manager"
author "doomhz"
start on startup
stop on shutdown
script
export HOME="/home/ubuntu"
echo $$ > /var/run/jslogger_manager.pid
cd /home/ubuntu/jslogger/; NODE_ENV=production /usr/local/bin/node /home/ubuntu/jslogger/app.js >> /home/ubuntu/log/upstart/jslogger_manager.sys.log 2>&1
end script
pre-start script
# Date format same as (new Date()).toISOString() for consistency
echo "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Starting" >> /home/ubuntu/log/upstart/jslogger_manager.sys.log
end script
pre-stop script
rm /var/run/jslogger_manager.pid
echo "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Stopping" >> /home/ubuntu/log/upstart/jslogger_manager.sys.log
end script
The ones for JSLogger Engine and SSL look the same.
After setting up the Upstart I can easily start and stop my app with:
sudo start jslogger_manager sudo stop jslogger_manager
Monit will guard your app and keep it running. This is the config:
#/etc/monit/conf.d/jslogger_manager
#!monit
set logfile /home/ubuntu/log/monit/jslogger_manager.log
check process nodejs with pidfile "/var/run/jslogger_manager.pid"
start program = "/sbin/start jslogger_manager"
stop program = "/sbin/stop jslogger_manager"
if failed port 5000 protocol HTTP
request /
with timeout 10 seconds
then restart
And for the Logger Engine:
#/etc/monit/conf.d/jslogger_logger
#!monit
set logfile /home/ubuntu/log/monit/jslogger_logger.log
check process nodejs_logger with pidfile "/var/run/jslogger_logger.pid"
start program = "/sbin/start jslogger_logger"
stop program = "/sbin/stop jslogger_logger"
if failed port 6987 protocol HTTP
request /
with timeout 10 seconds
then restart
Set up Monit demon to run a check each 60 seconds:
sudo monit -d 60 -c /etc/monit/monitrc
Start and stop your apps with Monit:
sudo monit stop all sudo monit start all
Btw, if you get an annoying error like “monit: Cannot connect to the monit daemon. Did you start it with http support?” when starting Monit, than you should check your configs in /etc/monit/monitrc and enable (uncomment) the http support:
set httpd port 2812 and
use address localhost # only accept connection from localhost
allow localhost # allow localhost to connect to the server and
allow admin:monit # require user 'admin' with password 'monit'
allow @monit # allow users of group 'monit' to connect (rw)
allow @users readonly # allow users of group 'users' to connect readonly
Routing HTTP traffic with HAProxy
It would be nice to make JSLogger run on port 80. There are a couple of good tools that I’ve already worked with and can do that: Apache, Nginx. First, I’ve installed Nginx to take care of routing the traffic from port 80 to 5000, where the JSLogger Manager is running. Everything seemed to be fine except that the Wesockets did not work anymore. SocketIO could not connect to my app anymore, it was always falling back to Ajax long polling.
Apparently Nginx can’t handle Websockets properly yet. There is a patch for it but it’s seemed to be buggy at that moment.
Luckily HAProxy exists, an amazing tool to route your traffic and that takes care of the Websockets. I followed this tutorial to install it.
Installing v 1.5 of HAProxy on Ubuntu:
# Install HAProxy cd ~ wget http://haproxy.1wt.eu/download/1.5/src/devel/haproxy-1.5-dev6.tar.gz tar xzf haproxy-1.5-dev6.tar.gz cd haproxy* sudo make install
The JSLogger HAProxy config looks like this:
#/etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local0 notice
maxconn 4096
daemon
#debug
#quiet
defaults
log global
mode http
option httplog
option dontlognull
retries 3
option redispatch
maxconn 2000
contimeout 5000
clitimeout 50000
srvtimeout 50000
frontend all 0.0.0.0:80
timeout client 86400000
default_backend jslogger_manager
acl is_websocket hdr(upgrade) -i websocket
acl is_websocket hdr_beg(host) -i ws
redirect prefix http://jslogger.com code 301 if { hdr(host) -i www.jslogger.com }
use_backend jslogger_manager if is_websocket
backend jslogger_manager
option forwardfor
timeout server 86400000
timeout connect 4000
server nodejs 127.0.0.1:5000 weight 1 maxconn 10000 check
Test and run HAProxy:
# Test haproxy config haproxy -c -f /etc/haproxy/haproxy.cfg # Run HAProxy haproxy -f /etc/haproxy/haproxy.cfg
Handling SSL with Stunnel
Next step was handling the SSL requests. Apparently HAproxy can’t do it. SSL support will be available only in the new version.
Stunnel takes care of it. I spent a couple of hours configuring Stunnel and HAProxy to work together. This was happening because I was following old tutorials. Stunnel didn’t play nicely with HAProxy before, it had to be patched to work. After spending a couple of hours patching old versions I’ve read the Stunnel manual 
They introduced a new PROXY protocol that solves the HAProxy integration.
Install Stunnel:
sudo apt-get install stunnel
Enable Stunnel (set ENABLED flag to 1):
# /etc/default/stunnel # Julien LEMOINE# September 2003 # Change to one to enable stunnel automatic startup ENABLED=1 FILES="/etc/stunnel/*.conf" OPTIONS="" # Change to one to enable ppp restart scripts
Configure Stunnel to route the JSLogger HTTPS traffic to HAproxy:
sslVersion = all options = NO_SSLv2 pid = /var/run/stunnel.pid socket = l:TCP_NODELAY=1 socket = r:TCP_NODELAY=1 output = /var/log/stunnel.log [https_default] accept = 443 connect = 80 cert = /home/ubuntu/jslogger/cert/ssl.pem key = /home/ubuntu/jslogger/cert/jslogger.com.key
Restart Stunnel:
sudo /etc/init.d/stunnel4 restart
At this moment the entire traffic from port 443 will be redirected to port 80.
Signed SSL certificate with StartSSL
We needed a valid, signed SSL certificate to make JSLogger serve the jslogger.js script from HTTPS without being blocked by the browser’s security alerts, as it happens when using a self-signed certificate.
I wanted to buy one but @cimm advised me that we don’t have to spend money on an expensive one at the moment, StartSSL can provide a trusted certificate that works fine in all major browsers. There is enough info on their site about how to generate and sign your own certificate.
All done
That’s the entire setup and deployment process of JSLogger. The next step will be to scale it smoothly when needed. HAproxy will take care of load balancing and switching to MongoHQ can easily solve the database scaling.
