Data You Can Punch

Have you been hurt by bad data? Do you want to hurt it back? With a little D3.js and Matter.js, now you can!

Have you ever opened a dataset to find out that every second value was missing? Were you forced to manually geo-code the last 2,000 addresses when you ran out of Google Maps API requests? Was your original thesis scrapped because every result turned out insignificant?

This is a plot for anyone who’s ever been hurt by bad data. Grab the “fist” below and vent your frustration.

Reset Simulation ↺

d3 = require("d3");
Matter = require("matter-js");

Engine = Matter.Engine
Bodies = Matter.Bodies
Composite = Matter.Composite

// Constants -------------------------------------------------------------------

// Mobile detection
is_mobile = /iPhone|iPad|iPod|Android/i.test(navigator.userAgent);

// Dimensions of the SVG viewbox and margins, defines our world-space
width = is_mobile ? (screen.width * 0.9) : 800;
height = is_mobile ? (screen.width * 0.9) : 400;
stroke_width = 1;

margin_left = 10;
margin_right = is_mobile ? 80 : 100; // Leave room for the "fist"
margin_top = 10;
margin_bottom = 10;

// Collision filters for Matter.js, note that these need to be powers of 2
collision_fist = 0x0001; 
collision_bar = 0x0002;
collision_mouse = 0x0004;

// Duration of D3 transitions (milliseconds)
duration_bar_fade_in = 1000;
duration_bar_fade_out = 1000;

// See here for information on using Matter.js collision filters. In this plot,
// we use them to only allow mouse interaction with the "fist" and not the bars
// of the bar plot.
//
// https://stackoverflow.com/questions/64772783/how-can-i-change-the-collisionfilter-of-an-object-so-it-can-no-longer-interact-w/73262781#73262781

// TODO: Make this a little more robust!
// 
// Generate 20 years of random data, ending last year
num_years = is_mobile ? 7 : 20;
scale = is_mobile ? 5 : 10;
offsets = d3.shuffle(d3.range(-10, 10)).slice(0, num_years);
this_year = new Date().getFullYear();
data = d3
  .range(this_year - num_years, this_year)
  .map((year, i) => ({
    label: year,
    // Fix the maximum value so that the y-axis always looks consistent
    value: (i == (num_years - 1)) ? (num_years - 1) * scale : Math.max((i * scale) + offsets[i], scale)
  }))

// Rotating plot titles (cycle on plot re-set)
plot_title_index = [0];
plot_titles = [
  "P-value of 0.055 (never published)",
  "Didn’t survive FDR",
  "Lasso selected none of my favourites",
  "Adding another hyperprior didn’t help",
  "P-value of 0.00001",
  "32 hour bootstrap"
];

// Helpers ---------------------------------------------------------------------

// Transform Matter body vertices into SVG <polygon> points
vertices_to_points = function(vertices) {
  return vertices.map(vertex => `${vertex.x},${vertex.y}`).join(" ")
}

// SVG -------------------------------------------------------------------------

// Move the z-index of all text backwards, it looks nicer when the physics
// objects appear above the rest of the article text.
d3.select("#title-block-header")
  .style("position", "relative")
  .style("z-index", -1);

d3.select(".nav-footer")
  .style("background", "rgba(0, 0, 0, 0)");

plot_title = d3.select("#plot-title")
  .append("h3")
  .text("Simulated Data")
  .style("opacity", 1)
  .style("position", "relative")
  .style("z-index", -1)
  // Prevent the cursor from highlighting text, this happens a lot while
  // dragging the "fist" around, causing small glitches.
  .style("user-select", "none");

// Set the SVG viewbox
svg = d3.select("#matter-container")
  // TODO: I've only gotten Matter.js to work with a fixed width/height 
  // container. Look into whether a reactive size is possible!
  // The mouse has an offset and scale, there's probably a way to allow
  // for resizing: https://brm.io/matter-js/docs/classes/Mouse.html
  //
  // Maybe helpful:
  // https://stackoverflow.com/questions/64302906/scaling-matter-js-canvas-breaks-mouse-drag
  // https://github.com/liabru/matter-js/issues/955
  // https://www.youtube.com/watch?v=kdSDTaYY700
  .style("width", width + "px")
  .style("height", height + "px")
  .append("svg")
  .attr("viewBox", [0, 0, width, height])
  .attr("style", "max-width: 100%; height: auto;")
  // Allows the physics objects to fall outside of the viewbox
  .attr("overflow", "visible");

// Initialize a container for the plot bars and "fist". Note that we initialize
// the fist container and prompt with an explicitly opacity of 1, otherwise, 
// when we later use `.transition()` to fade these elements out, their opacity
// attribute is instantly added, causing the element to "blink".
bar_container = svg.append("g").classed("bar-container", true)
fist_container = svg.append("g").classed("fist-container", true).attr("opacity", 1)
fist_prompt_container = svg.append("g").attr("id", "fist-prompt").attr("opacity", 1)

// Scales
x_axis = d3.scaleBand()
  .domain(data.map(d => d.label))
  .range([margin_left, width - margin_right])
  .padding(0.2);

y_axis = d3.scaleLinear()
  .domain([0, d3.max(data, d => d.value)])
  .range([height - margin_top, margin_bottom]);
  
// Add the x and y axes
svg.append("g")
  .classed("x-axis", true)
  // Account for the stroke width of both the bars and the axis itself
  .attr("transform", `translate(0,${height - margin_bottom + (stroke_width)})`)
  .call(d3.axisBottom(x_axis).tickSizeOuter(0))
  // Extend the right-side of the axis line to account for the right margin.
  // This gives a spot for the `fist` to land. We have to manually alter the `d`
  // attribute of the <path>.
  .call(g => {
    const old_path = g.select(".domain").attr("d");
    const new_path = old_path.replace(/H(\d+)/, (match, p1) => `H${+p1 + margin_right}`);
    g.select(".domain").attr("d", new_path);
  })
  .call(g => g.select(".domain").attr("stroke-width", stroke_width))
  .call(g => g.selectAll("line").attr("stroke-width", stroke_width))
  .call(g => g.append("text")
    .attr("x", width - margin_right)
    // Grab the y-position from the existing tick labels, so this is aligned
    .attr("y", g.select(".tick text").attr("y"))
    .attr("dy", g.select(".tick text").attr("dy"))
    .attr("fill", "currentColor")
    .attr("text-anchor", "start")
    .text("Year →")
  )
  // Set the axis font to the website default and prevent user selection of text
  .call(g => g.selectAll("text")
    .style("font-family", "var(--bs-body-font-family)")
    .style("user-select", "none")
  )
  .lower();

svg.append("g")
  .classed("y-axis", true)
  // `+ 1` to connect the axis tick at `0` with the x-axis line
  .attr("transform", `translate(${margin_left + 1}, ${stroke_width})`)
  .call(d3.axisLeft(y_axis))
  .call(g => g.select(".domain").remove())
  .call(g => g.selectAll("line").attr("stroke-width", stroke_width))
  .call(g => g.append("text")
    .attr("x", -margin_left)
    .attr("y", d3.min(y_axis.range()) - (is_mobile ? 15 : 10))
    .attr("fill", "currentColor")
    .attr("text-anchor", "middle")
    .text("↑ Value")
  )
  // Set the axis font to the website default and prevent user selection of text
  .call(g => g.selectAll("text")
    .style("font-family", "var(--bs-body-font-family)")
    .style("user-select", "none")
  )
  .lower();

engine = Engine.create();
engine.gravity.scale = 0.001;

engine.gravity.y = 1;

engine.gravity.x = 0;

bodies = engine.world.bodies;

// Create an invisible ground for the bars to sit on. The width shouldn't matter
// that much but I imagine thicker is a little better for collision detection.
// The y-position is important to get right so that the bars sit right ontop of
// the lower margin (`margin_bottom`).
ground_width = 10;
ground = Bodies.rectangle(
  width / 2, height - (margin_bottom / 2), width, margin_bottom,
  { 
    isStatic: true, 
    label: "ground"
  }
);
Composite.add(engine.world, ground);

mouse = Matter.Mouse.create(document.querySelector("#matter-container"));
// mouse.pixelRatio = window.devicePixelRatio; // TODO: Figure this out
mouse_constraint = Matter.MouseConstraint.create(engine, {
  mouse: mouse,
  collisionFilter: {
    category: collision_mouse,
    mask: collision_fist
  },
  constraint: { stiffness: 0.5 } // Raising stiffness makes drag less "springy"
})
Composite.add(engine.world, mouse_constraint);

create_bars = function(first = false) {
  
  // console.log(bodies[1].mass)
  // console.log(bodies[1])
  
  // Create bodies corresponding to each bar in bar chart
  bar_container
    .selectAll("bar")
    .data(data)
    // Initialize a <polygon> for each bar. We'll use the vertices
    // provided by Matter to draw these later.
    .join("polygon")
    .classed("plot-data-blue", true)
    .attr("stroke-width", stroke_width)
    .attr("data-label", d => d.label)
    // Set the opacity to 0, so we can fade the bars in
    .attr("opacity", first ? 1 : 0)
    // Initialize a Matter physics body for each bar. Note `this` means something
    // different if you using a `=>` function here - be aware!
    .each(function(d, i) {
      
      const x = x_axis(d.label);
      const bar_width = x_axis.bandwidth();
      const bar_height = height - margin_bottom - y_axis(d.value);
      
      // Matter expects centered x, y coordinates whereas SVG and D3 expects
      // coordinates relative to the top-left.
      const body = Bodies.rectangle(
        // Center relative to the full band-width, to put each bar in the middle
        x + (bar_width / 2),
        height - margin_bottom - (bar_height / 2),
        bar_width, 
        bar_height,
        {
          collisionFilter: {
            category: collision_bar,
            // Collide with other bars or the "fist", note that this is a bit-mask
            mask: collision_fist | collision_bar
          },
          label: "bar"
        }
      );
      Composite.add(engine.world, body);
      
      // Synchronize the D3 and Matter IDs and positions
      d3.select(this)
        .attr("id", `body-${body.id}`)
        .attr("data-body-id", body.id)
        .attr("points", vertices_to_points(body.vertices))
        .raise()
    });
    
    // Fade in all of the bars on re-set
    if (!first) {
      svg
        .select(".bar-container")
        .selectAll("polygon")
        .transition()
        .duration(duration_bar_fade_in)
        .attr("opacity", 1)
    }
}

// Remove the bar chart physics bodies and fade them to the screen
remove_bars = function() {
  bar_container
    .selectAll("polygon")
    .transition()
    .duration(duration_bar_fade_out)
    .attr("opacity", 0)
    // Remove the bar and it's physics body
    .on("end", function() { 
      const body_id = d3.select(this).attr("data-body-id");
      Composite.remove(engine.world, Composite.get(engine.world, body_id, "body"));
      d3.select(this).remove(); 
    })
}

// Create the "fist" physics body and render it to the screen
create_fist = function(first = false) {
  
  // Slightly smaller diameter than the margin
  const fist_radius = (margin_right / 2) * 0.8;
  
  // If `first` the ball is sitting on the ground, otherwise we drop it from
  // the sky.
  const fist_y = first ? height - (margin_bottom / 2) - (fist_radius / 2) : 0;
  
  // Create a circular body for the "fist"
  const fist = Bodies.polygon(
    width - (margin_right / 2), // Placed in the middle of the right margin
    fist_y,
    50, // 50 sided
    fist_radius,
    {
      collisionFilter: {
        category: collision_fist,
        mask: collision_fist | collision_bar | collision_mouse
      }
    }
  );
  
  Composite.add(engine.world, fist);
  
  // Render the Fist
  fist_container
    .attr("opacity", first ? 1 : 0)
    .call(g => g.append('polygon')
      .attr("id", `body-${fist.id}`)
      .attr("data-body-id", fist.id)
      .attr("points", vertices_to_points(fist.vertices))
      .classed("plot-data-black", true)
      .attr("stroke-width", stroke_width)
      // Change the cursor icon while interacting with the fist to make it more
      // obvious that you can drag it.
      .style("cursor", "grab")
      .on("mousedown", function() {
        d3.select(this).style("cursor", "grabbing");
        // Remove the user-prompt to pick up the fist once they've grabbed it
        d3.select("#fist-prompt")
          .attr("pointer-events", "none")
          .transition()
          .duration(1000)
          .style("opacity", 0)
          .on("end", function() { d3.select(this).remove(); })
      })
      .on("mouseup", function() {
        d3.select(this).style("cursor", "grab");
      })
    )
    .call(g => g.append("text")
      .classed("plot-annotation", true)
      .attr("id", `body-text-${fist.id}`)
      .attr("x", fist.position.x)
      .attr("y", fist.position.y)
      .attr("text-anchor", "middle") // Centers text horizontally
      .attr("dominant-baseline", "middle") // Centers text vertically
      .attr("font-size", 16)
      .attr("font-weight", 400)
      .text("fist")
      // Prevents trying to copy/highlight the text, which happens a lot on drag
      .style("user-select", "none")
      .style("pointer-events", "none")
    )
    .raise();
    
  if (!first) {
    fist_container
      .transition()
      .duration(duration_bar_fade_in)
      .attr("opacity", 1)
  }
    
  if (first) {
    // Add a label above the fist prompting the user to click on it. We create the
    // "fist" later using `create_fist()`.
    fist_prompt_container
      .call(g => g.append("text")
        .classed("plot-annotation", true)
        .attr("x", fist.position.x)
        .attr("y", fist.position.y - fist_radius - 80)
        .attr("text-anchor", "middle")
        .attr("font-size", 16)
        .attr("font-weight", 400)
        .text("Pick Up")
      )
      .call(g => g.append("text")
        .classed("plot-annotation", true)
        .attr("x", fist.position.x)
        .attr("y", fist.position.y - fist_radius - 80 + 18)
        .attr("text-anchor", "middle")
        .attr("font-size", 16)
        .attr("font-weight", 400)
        .text("↓")
      )
  }
}

// Remove the "fist" physics body and it from the screen
remove_fist = function() {
  fist_container
    .transition()
    .duration(duration_bar_fade_out)
    .attr("opacity", 0)
    // Remove any child elements of the container (i.e. "fist" and it's label)
    // and the corresponding physics body.
    .on("end", function() { 
      const fist_id = d3.select(this).select("polygon").attr("data-body-id");
      Composite.remove(engine.world, Composite.get(engine.world, fist_id, "body"));
      d3.select(this).selectAll("*").remove(); 
    });
    
  // Remove the "fist" interaction prompt if it still exists
  fist_prompt_container
    .transition()
    .duration(duration_bar_fade_out)
    .attr("opacity", 0)
    .on("end", function() { d3.select(this).remove(); });
}

// When the category selector is changed, filter the posts and record the
// selection in `previous_selection`.
// mutable click_off = false;
d3.select("#reset-button")
  .style("user-select", "none")
  .on("click", function() {
    // TODO: Think of a much more robust way to prevent spamming of this
    // button. I think on each regeneration we should destroy pause the physics
    // and then destroy every physics body immediately. Then we un-pause at the
    // end of this callback.
    
    // TODO: Nice transition for incrementing the plot title
  
    // Prevent interaction with this button
    const button = d3.select(this).style("pointer-events", "none");
    
    // Increment the plot title, fading out and then back in
    plot_title
      .transition()
      .duration(duration_bar_fade_out)
      .style("opacity", 0)
      .on("end", function() {
        plot_title.text(plot_titles[plot_title_index[0]]);
        plot_title_index[0] = (plot_title_index[0] + 1) % plot_titles.length;
        plot_title.transition().duration(duration_bar_fade_in).style("opacity", 1);
      });
    
    // Remove physics bodies
    // clear_bodies();
    remove_bars();
    remove_fist();
    
    // Wait until the end of the `remove_*()` transition to re-generate the
    // bars and fist.
    svg
      .transition()
      .delay(duration_bar_fade_out + 100)
      .on("end", function() {
        create_bars();
        create_fist();
      });
    
    // Re-activate pointer events using this button
    svg
      .transition()
      .delay(duration_bar_fade_in + duration_bar_fade_out + 500)
      .on("end", function() { button.style("pointer-events", "auto"); });
  });

create_bars(true);

create_fist(true);

function* update() {
  
  while (true) {
    // Update the engine
    Engine.update(engine, 5)
    
    // TODO: We could remove bodies (via Matter) and polygons (via D3) once
    // they've left the screen to get some more performance. Doesn't really
    // matter for this demo.
    
    // TODO: Might be fun to add a gravity slider in the future, but I'm
    // trying to avoid packing every feature into this plot.
    
    // Move the vertices of every body to their new positions. This is the 
    // SVG equivalent of Matter's example <canvas> render loop:
    // https://github.com/liabru/matter-js/wiki/Rendering
    bodies.filter(body => body.label != "ground").forEach(body => {
      svg.select(`#body-${body.id}`)
        .attr('points', vertices_to_points(body.vertices));
      
      // Rotate any text labels attached to a physics body to match the body's
      // rotation. Note that the Matter.js body angles are in radians, but
      // `rotate()` expects degrees.
      const body_angle = body.angle * (180 / Math.PI);
      const body_x = body.position.x;
      const body_y = body.position.y;
      svg.select(`#body-text-${body.id}`)
        .attr("x", body_x)
        .attr("y", body_y)
        .attr("transform", `rotate(${body_angle}, ${body_x}, ${body_y})`);
    })
    
    // Finish
    yield
  }
}

// Call the `update()` function once to begin the loop
update();