4/10/2014 Processing Delay Postmortem

every

  apt-get update
  apt-get upgrade
  openssl version -a  # should show that it was built on April 7, 2014
  service nginx restart

1
2
3
4

_rollbarParams = {
    // ... other configuration
    "client.javascript.code_version": "bdd2b9241f791fc9f134fb3244b40d452d2d7e35"
}

1
2
3
4

Rollbar.configure do |config|
    # ... other configuration
    config.code_version = 'bdd2b9241f791fc9f134fb3244b40d452d2d7e35'
end

1

tail -f /var/log/haproxy.log | awk '{print $11,$0}' | grep '^5' | awk '{$1="";print "warning",$0}' | rollbar -t $ACCESS_TOKEN -e production -v

1

tail -f /var/log/auth.log | grep -i 'Failed password' | awk '{print "error user ",$11,"failed auth from ",$13}' | rollbar -t $ACCESS_TOKEN -e ops

1
2
3
4
5
6
7
8
9
10
11
12
13
14

<script>
!function(r,n,e){var t={"notifier.plugins.jquery.version":"0.0.1"};n._rollbar.push(
{_rollbarParams:t});r(e).ajaxError(function(r,e,t,u){var o=e.status;var a=t.url;
n._rollbar.push({level:"warning",msg:"jQuery ajax error for url "+a,jquery_status:o,
jquery_url:a,jquery_thrown_error:u,jquery_ajax_error:true})});var u=r.fn.ready;
r.fn.ready=function(r){return u.call(this,function(){try{r()}catch(e){
n._rollbar.push(e)}})};var o={};var a=r.fn.on;r.fn.on=function(r,e,t,u){
var f=function(r){var e=function(){try{return r.apply(this,arguments)}catch(e){
n._rollbar.push(e);return null}};o[r]=e;return e};if(e&&typeof e==="function"){
e=f(e)}else if(t&&typeof t==="function"){t=f(t)}else if(u&&typeof u==="function"){
u=f(u)}return a.call(this,r,e,t,u)};var f=r.fn.off;r.fn.off=function(r,n,e){
if(n&&typeof n==="function"){n=o[n];delete o[n]}else{e=o[e];delete o[e]}
return f.call(this,r,n,e)}}(jQuery,window,document);
</script>

1

"notifier.plugins.jquery.ignoreAjaxErrors": true

...
app/
    ...
    views.py
    static/
        js/
            site.js (containing various models and functionality used in your app)
            jquery.js (unminified)
            util.js
    templates/
        index.html

1
2
3
4
5
6
7
8

var aFunction = function() {
    var a = b;
}

App = {};
App.errorCausingFunction = function() {
    aFunction();
}

1
2
3
4
5
6
7
8
9

...
<script src="static/js/site.js"></script>
<script src="static/js/jquery.js"></script>
<script src="static/js/util.js"></script>
....
<script>
    App.errorCausingFunction();
</script>
...

npm install -g uglify-js

uglifyjs static/js/site.js static/js/jquery.js static/js/util.js --output static/js/all.min.js

1

<script src="static/js/all.min.js"></script>

uglifyjs static/js/site.js static/js/jquery.js static/js/util.js --output static/js/all.min.js --source-map static/js/all.min.map --source-map-url /static/js/all.min.map

1
2
3
4
5
6
7

_rollbarParams = {
  // ... other params ...
  // set this to 'true' to enable source map processing
"client.javascript.source_map_enabled": true,
  // provide the current code version, i.e. the git SHA of your javascript code.
"client.javascript.code_version": "bdd2b9241f791fc9f134fb3244b40d452d2d7e35"
}

1
2

// ... minified js file contents ...
//# sourceMappingURL=<url for source map>

1
2
3
4
5
6

curl https://api.rollbar.com/api/1/sourcemap \
-F access_token=aaaabbbbccccddddeeeeffff00001111 \
-F version=bdd2b9241f791fc9f134fb3244b40d452d2d7e35 \
-F minified_url=http://127.0.0.1:8005/static/js/all.min.js \
-F source_map=@app/static/js/all.min.map \
-F app/static/js/site.js=@app/static/js/site.js

1
2
3
4
5
6
7

_rollbarParams = {
  // ... existing params ...
  // set this to 'true' to enable source map processing
  "client.javascript.source_map_enabled": true,
  // provide the current code version, i.e. the git SHA of your javascript code.
  "client.javascript.code_version": "bdd2b9241f791fc9f134fb3244b40d452d2d7e35"
}

1
2
3
4
5

try {
  doSomething();
} catch (e) {
  _rollbar.push(e);
}

1
2

//Mocha
[1,2,3].indexOf(5).should.equal(-1);

1
2

//Vows
assert.equal([1,2,3].indexOf(5), -1);

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

var vows = require('vows');
var assert = require('assert');
vows.describe('testmodule').addBatch({
  "call username() with a valid user id": {
    topic: function() {
      var callback = this.callback;
      return username(42, this.callback);
    },
    "and verify username is correct": function(err, username) {
      assert.isNull(err);
      assert.isString(username);
      assert.equal(username, "cory");
    }
  }
}).export(module, {error: false});

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

exports.initTestingAppChildProc = function(config, promise) {
  // ... Setup temporary config file
  // ... Get path to your main app.js
  // ... Initialize the api library

  // fork a child process to start the api server
  var args = [configPath, 'test'];
  var appProc = fork(appJsPath, args);

  // This is used to tell if our API server died during its
  // initialization.
  var pendingCallback = true;

  appProc.on('message', function(message) {
    if (message == 'ready') {
      pendingCallback = false;

      // This is how we know our API server is ready to 
      // receive requests. The message is emitted in the
      // API server once it's ready to receive requests.
      promise.emit('success', null, appProc);
    }
  });
  appProc.on('exit', function(code, signal) {
    if (pendingCallback) {
      var msg = 'child process exited before callback';
      console.error(msg);
      promise.emit('error', new Error(msg));
    }
  });
  appProc.on('SIGTERM', function() {
    process.exit();
  });
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

// initialize the API and start the web server when it's ready
api.init(config, function(err) {
  if (err) {
    log.error('Could not initialize API: ' + err);
    process.exit(1);
  } else {
    // Start up the server
    var httpServer = app.listen(port, host, function() {
      log.info('API server is ready.');
      log.info('Listening on ' + host + ':' + port);

      // Use the "ready" message to signal that the server is ready.
      // This is used by the test suite to wait for the api server
      // process to start up before sending requests.
      if (process.send) {
        process.send('ready');
      }
    });
  }
});

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

vows.describe('routes.project').addBatch({
  // Provides a reference to the api server child process
  'Start up an API server': {
    topic: function() {
      var promise = new events.EventEmitter();
      common.initTestingAppChildProc(config, promise);
      return promise;
    },
    teardown: function(err, childProc) {
      var callback = this.callback;
      var shutdown = function() {
        api.shutdown(callback);
      };
      if (childProc) {
        childProc.on('exit', function(code, sig) {
          shutdown();
        });
        childProc.kill();
      } else {
        shutdown();
      }
    },
    'and get a valid project': {
      topic: function(err, childProc) {
        common.apiGet(url('api/1/project/',
            {access_token: config.test.validEnabledReadAccessToken}), this.callback);
      },
      'returns 200 OK': common.assertStatus(200),
      'returns JSON': common.assertJsonContentType(),
      'fast local response time': common.assertMaxResponseTime(20),
      "returns a valid api response": common.assertValidApiResponse(),
      "has a result key in the JSON response": common.assertJsonHasFields(['result']),
      "there's no api error": common.assertNoApiError(),
      'all of the deploy fields are available': common.assertJsonHasFields(db.projectFields(),
          'result'),
      'cross-check account id with api.getAccount': {
        topic: function(err, resp, body) {
          var project = body.result;
          api.getAccount(body.result.account_id, this.callback);
        },
        'verify the account is not null': function(err, account) {
          assert.isNull(err);
          assert.isObject(account);
        }
      }
    }
}).export(module, {error: false});

1

✓ OK » 497 honored (33.232s)

1

gem 'rollbar', '~> 0.9.14'

The Planned Maintenance

Timeline

Follow-on Tasks

Conclusion

What is Heartbleed?

Are you affected?

How We Responded

Recommended actions for Rollbar Customers

Note for Heroku Users

Closing notes

Auto-resolving items in GitHub commits

Local Debugging with Source Maps

Production Debugging with Source Maps

Automating things

More info

What Are Source Maps?

Source Maps and Rollbar

Getting this set up

Caveats

We use Vows. Why not Mocha?

A simple Vows test

Useful design patterns (I swear this will be short)

Separate your view logic from your API business logic

Make every function you write use a callback.

Make the first argument to every callback be an optional error.

Testing the API server, for reals

Gotchas

Wrapping up…

Moment of zen

Big Features

Small Features

Library Updates

Ruby

Python

Bug Fixes

Documentation Updates