Data

This page has two tools, and individual tables have their own pages:

Explore all datasets — filter and browse across all IRW datasets by size, response type, and other properties. As you filter, this tool builds a ready-to-run R code snippet that reproduces your selection with the irw package.
Look up a single dataset — search for one dataset by name and generate its citation (reference and BibTeX) for use in your own work.
Browse table pages — every table with a recorded licence has its own page, giving a description of what it measures and where it came from, its size and shape (responses, respondents, items and response categories), the tags that classify it, item text statistics where they exist, the columns it contains, and ready-to-run R and Python code to fetch it. Each page names the IRW version it describes, so it can be cited.

For what the filter columns mean, see Metadata. Ready-made groups of tables (randomized trials, Big Five, and so on) are listed under Collections, and every dataset’s description, licence and source are listed in one place on Dataset documentation.

Explore all datasets

Code

function interval(range = [], options = {}) {
  const [min = 0, max = 1] = range;
  const {
    step = .001,
    label = null,
    value = [min, max],
    format = ([start, end]) => `${start} … ${end}`,
    color,
    width,
    theme,
    __ns__ = randomScope(),
  } = options;

  const css = `
#${__ns__} {
  //font: 13px/1.2 var(--sans-serif);
  display: flex;
  align-items: baseline;
  flex-wrap: wrap;
  max-width: 100%;
  width: auto;
  flex-direction: column;
}
@media only screen and (min-width: 30em) {
  #${__ns__} {
    flex-wrap: nowrap;
    width: 360px;
  }
}
#${__ns__} .label {
  //width: 120px;
  //padding: 5px 0 4px 0;
  //padding: 5px 0 0 0;
  //margin-right: 6.5px;
  flex-shrink: 0;
}
#${__ns__} .form {
  display: flex;
  width: 100%;
}
#${__ns__} .range {
  flex-shrink: 1;
  width: 100%;
}
#${__ns__} .range-slider {
  width: 100%;
}
  `;
  
  const $range = rangeInput({min, max, value: [value[0], value[1]], step, color, width, theme});
  const $output = html`<output>`;
  const $view = html`<div id=${__ns__}>
${label == null ? '' : html`<div class="label">${label}`}
<div class=form>
  <div class=range>
    ${$range}<div class=range-output>${$output}</div>
  </div>
</div>
${html`<style>${css}`}
  `;

  const update = () => {
    const content = format([$range.value[0], $range.value[1]]);
    if(typeof content === 'string') $output.value = content;
    else {
      while($output.lastChild) $output.lastChild.remove();
      $output.appendChild(content);
    }
  };
  $range.oninput = update;
  update();
  
  return Object.defineProperty($view, 'value', {
    get: () => $range.value,
    set: ([a, b]) => {
      $range.value = [a, b];
      update();
    },
  });
}

function rangeInput(options = {}) {
  const {
    min = 0,
    max = 100,
    step = 'any',
    value: defaultValue = [min, max],
    color,
    width,
    theme = theme_Flat,
  } = options;
  
  const controls = {};
  const scope = randomScope();
  const clamp = (a, b, v) => v < a ? a : v > b ? b : v;

  // Will be used to sanitize values while avoiding floating point issues.
  const input = html`<input type=range ${{min, max, step}}>`;
  
  const dom = html`<div class=${`${scope} range-slider`} style=${{
    color,
    width: cssLength(width),
  }}>
  ${controls.track = html`<div class="range-track">
    ${controls.zone = html`<div class="range-track-zone">
      ${controls.range = html`<div class="range-select" tabindex=0>
        ${controls.min = html`<div class="thumb thumb-min" tabindex=0>`}
        ${controls.max = html`<div class="thumb thumb-max" tabindex=0>`}
      `}
    `}
  `}
  ${html`<style>${theme.replace(/:scope\b/g, '.'+scope)}`}
</div>`;

  let value = [], changed = false;
  Object.defineProperty(dom, 'value', {
    get: () => [...value],
    set: ([a, b]) => {
      value = sanitize(a, b);
      updateRange();
    },
  });

  const sanitize = (a, b) => {
    a = isNaN(a) ? min : ((input.value = a), input.valueAsNumber);
    b = isNaN(b) ? max : ((input.value = b), input.valueAsNumber);
    return [Math.min(a, b), Math.max(a, b)];
  }
  
  const updateRange = () => {
    const ratio = v => (v - min) / (max - min);
    dom.style.setProperty('--range-min', `${ratio(value[0]) * 100}%`);
    dom.style.setProperty('--range-max', `${ratio(value[1]) * 100}%`);
  };

  const dispatch = name => {
    dom.dispatchEvent(new Event(name, {bubbles: true}));
  };
  const setValue = (vmin, vmax) => {
    const [pmin, pmax] = value;
    value = sanitize(vmin, vmax);
    updateRange();
    // Only dispatch if values have changed.
    if(pmin === value[0] && pmax === value[1]) return;
    dispatch('input');
    changed = true;
  };
  
  setValue(...defaultValue);
  
  // Mousemove handlers.
  const handlers = new Map([
    [controls.min, (dt, ov) => {
      const v = clamp(min, ov[1], ov[0] + dt * (max - min));
      setValue(v, ov[1]);
    }],
    [controls.max, (dt, ov) => {
      const v = clamp(ov[0], max, ov[1] + dt * (max - min));
      setValue(ov[0], v);
    }],
    [controls.range, (dt, ov) => {
      const d = ov[1] - ov[0];
      const v = clamp(min, max - d, ov[0] + dt * (max - min));
      setValue(v, v + d);
    }],
  ]);
  
  // Returns client offset object.
  const pointer = e => e.touches ? e.touches[0] : e;
  // Note: Chrome defaults "passive" for touch events to true.
  const on  = (e, fn) => e.split(' ').map(e => document.addEventListener(e, fn, {passive: false}));
  const off = (e, fn) => e.split(' ').map(e => document.removeEventListener(e, fn, {passive: false}));
  
  let initialX, initialV, target, dragging = false;
  function handleDrag(e) {
    // Gracefully handle exit and reentry of the viewport.
    if(!e.buttons && !e.touches) {
      handleDragStop();
      return;
    }
    dragging = true;
    const w = controls.zone.getBoundingClientRect().width;
    e.preventDefault();
    handlers.get(target)((pointer(e).clientX - initialX) / w, initialV);
  }
  
  
  function handleDragStop(e) {
    off('mousemove touchmove', handleDrag);
    off('mouseup touchend', handleDragStop);
    if(changed) dispatch('change');
  }
  
  invalidation.then(handleDragStop);
  
  dom.ontouchstart = dom.onmousedown = e => {
    dragging = false;
    changed = false;
    if(!handlers.has(e.target)) return;
    on('mousemove touchmove', handleDrag);
    on('mouseup touchend', handleDragStop);
    e.preventDefault();
    e.stopPropagation();
    
    target = e.target;
    initialX = pointer(e).clientX;
    initialV = value.slice();
  };
  
  controls.track.onclick = e => {
    if(dragging) return;
    changed = false;
    const r = controls.zone.getBoundingClientRect();
    const t = clamp(0, 1, (pointer(e).clientX - r.left) / r.width);
    const v = min + t * (max - min);
    const [vmin, vmax] = value, d = vmax - vmin;
    if(v < vmin) setValue(v, v + d);
    else if(v > vmax) setValue(v - d, v);
    if(changed) dispatch('change');
  };
  
  return dom;
}

function randomScope(prefix = 'scope-') {
  return prefix + (performance.now() + Math.random()).toString(32).replace('.', '-');
}

cssLength = v => v == null ? null : typeof v === 'number' ? `${v}px` : `${v}`

html = htl.html

theme_Flat = `
/* Options */
:scope {
  color: #3b99fc;
  width: 240px;
}

:scope {
  position: relative;
  display: inline-block;
  --thumb-size: 15px;
  --thumb-radius: calc(var(--thumb-size) / 2);
  //padding: var(--thumb-radius) 0;
  margin: 2px;
  vertical-align: middle;
}
:scope .range-track {
  box-sizing: border-box;
  position: relative;
  height: 7px;
  background-color: hsl(0, 0%, 80%);
  overflow: visible;
  border-radius: 4px;
  padding: 0 var(--thumb-radius);
}
:scope .range-track-zone {
  box-sizing: border-box;
  position: relative;
}
:scope .range-select {
  box-sizing: border-box;
  position: relative;
  left: var(--range-min);
  width: calc(var(--range-max) - var(--range-min));
  cursor: ew-resize;
  background: currentColor;
  height: 7px;
  border: inherit;
}
/* Expands the hotspot area. */
:scope .range-select:before {
  content: "";
  position: absolute;
  width: 100%;
  height: var(--thumb-size);
  left: 0;
  top: calc(2px - var(--thumb-radius));
}
:scope .range-select:focus,
:scope .thumb:focus {
  outline: none;
}
:scope .thumb {
  box-sizing: border-box;
  position: absolute;
  width: var(--thumb-size);
  height: var(--thumb-size);

  background: #fcfcfc;
  top: -4px;
  border-radius: 100%;
  border: 1px solid hsl(0,0%,55%);
  cursor: default;
  margin: 0;
}
:scope .thumb:active {
  box-shadow: inset 0 var(--thumb-size) #0002;
}
:scope .thumb-min {
  left: calc(-1px - var(--thumb-radius));
}
:scope .thumb-max {
  right: calc(-1px - var(--thumb-radius));
}
`

Code

function hist(values = [], opt = {}) {
  const max = d3.max(values);
  let {
    domain,
    thresholds,
    width,
    height,
    fillStyle,
    strokeStyle,
    domainLineWidth
  } = {
    domain: [0, max],
    thresholds: d3.thresholdFreedmanDiaconis(
      values,
      ...(opt.domain ?? [0, max])
    ),
    width: 100,
    height: 14,
    fillStyle: 'rgba(0,0,0,1)',
    strokeStyle: 'rgba(0,0,0,0.2)',
    domainLineWidth: 1,
    ...opt
  };

  // custom threshold generator so we don't nice domains
  const thresholdGen = (data, min, max) =>
    d3.range(thresholds).map(t => min + (t / thresholds) * (max - min));

  const x = d3
    .scaleLinear()
    .domain(domain)
    .range([0, width]);
  const binsGenerator = d3
    .histogram()
    .domain(x.domain())
    .thresholds(thresholdGen);

  const bins = binsGenerator(values);

  const y = d3
    .scaleLinear()
    .domain([0, d3.max(bins.map(b => b.length))])
    .range([height - domainLineWidth, 0]);

  const ctx = DOM.context2d(width, height);

  ctx.fillStyle = fillStyle;

  if (domainLineWidth) {
    ctx.strokeStyle = strokeStyle;
    ctx.lineWidth = domainLineWidth;

    ctx.beginPath();
    ctx.moveTo(0, height - domainLineWidth / 2);
    ctx.lineTo(width, height - domainLineWidth / 2);
    ctx.stroke();
  }

  for (const d of bins) {
    ctx.fillRect(
      x(d.x0),
      y(d.length),
      x(d.x1) - x(d.x0) - 1,
      height - domainLineWidth - y(d.length)
    );
  }

  return ctx.canvas;
}

Code

tol = ({
    QualBright: ['#4477AA', '#EE6677', '#228833', '#CCBB44', '#66CCEE','#AA3377'],
    QualHighContrast: ['#004488', '#DDAA33', '#BB5566'],
    QualVibrant: ['#EE7733', '#0077BB', '#33BBEE', '#EE3377', '#CC3311', '#009988'],
    QualMuted: ['#CC6677', '#332288', '#DDCC77', '#117733', '#88CCEE','#882255', '#44AA99', '#999933', '#AA4499'],
    QualLight: ['#77AADD', '#EE8866', '#EEDD88', '#FFAABB', '#99DDFF', '#44BB99', '#BBCC33', '#AAAA00'],
    Sunset: ['#364B9A', '#4A7BB7', '#6EA6CD', '#98CAE1', '#C2E4EF', '#EAECCC', '#FEDA8B', '#FDB366', '#F67E4B', '#DD3D2D', '#A50026'],
    BuRd: ['#2166AC', '#4393C3', '#92C5DE', '#D1E5F0', '#F7F7F7', '#FDDBC7', '#F4A582', '#D6604D', '#B2182B'],
    PRGn: ['#762A83', '#9970AB', '#C2A5CF', '#E7D4E8', '#F7F7F7', '#D9F0D3', '#ACD39E', '#5AAE61', '#1B7837'],
    YlOrBr: ['#FFFFE5', '#FFF7BC', '#FEE391', '#FEC44F', '#FB9A29',
                '#EC7014', '#CC4C02', '#993404', '#662506'],
    Iridescent: ['#FEFBE9', '#FCF7D5', '#F5F3C1', '#EAF0B5', '#DDECBF',
                '#D0E7CA', '#C2E3D2', '#B5DDD8', '#A8D8DC', '#9BD2E1',
                '#8DCBE4', '#81C4E7', '#7BBCE7', '#7EB2E4', '#88A5DD',
                '#9398D2', '#9B8AC4', '#9D7DB2', '#9A709E', '#906388',
                '#805770', '#684957', '#46353A'],
    RainbowPuRd: ['#6F4C9B', '#6059A9', '#5568B8', '#4E79C5', '#4D8AC6',
                '#4E96BC', '#549EB3', '#59A5A9', '#60AB9E', '#69B190',
                '#77B77D', '#8CBC68', '#A6BE54', '#BEBC48', '#D1B541',
                '#DDAA3C', '#E49C39', '#E78C35', '#E67932', '#E4632D',
                '#DF4828', '#DA2222'],
    RainbowPuBr: ['#6F4C9B', '#6059A9', '#5568B8', '#4E79C5', '#4D8AC6',
                '#4E96BC', '#549EB3', '#59A5A9', '#60AB9E', '#69B190',
                '#77B77D', '#8CBC68', '#A6BE54', '#BEBC48', '#D1B541',
                '#DDAA3C', '#E49C39', '#E78C35', '#E67932', '#E4632D',
                '#DF4828', '#DA2222', '#B8221E', '#95211B', '#721E17',
                '#521A13'],
    RainbowWhRd: ['#E8ECFB', '#DDD8EF', '#D1C1E1', '#C3A8D1', '#B58FC2',
                '#A778B4', '#9B62A7', '#8C4E99', '#6F4C9B', '#6059A9',
                '#5568B8', '#4E79C5', '#4D8AC6', '#4E96BC', '#549EB3',
                '#59A5A9', '#60AB9E', '#69B190', '#77B77D', '#8CBC68',
                '#A6BE54', '#BEBC48', '#D1B541', '#DDAA3C', '#E49C39',
                '#E78C35', '#E67932', '#E4632D', '#DF4828', '#DA2222'],
    RainbowDiscrete: ['#E8ECFB', '#D9CCE3', '#D1BBD7', '#CAACCB', '#BA8DB4',
                '#AE76A3', '#AA6F9E', '#994F88', '#882E72', '#1965B0',
                '#437DBF', '#5289C7', '#6195CF', '#7BAFDE', '#4EB265',
                '#90C987', '#CAE0AB', '#F7F056', '#F7CB45', '#F6C141',
                '#F4A736', '#F1932D', '#EE8026', '#E8601C', '#E65518',
                '#DC050C', '#A5170E', '#72190E', '#42150A']
})

Code

Plot = import("https://cdn.jsdelivr.net/npm/@observablehq/plot@0.6/+esm")
import {yamultiselect} from '@saneef/yet-another-multi-select'

md_tagged = transpose(datasets_tagged)
md_notags = transpose(datasets_all)
md = use_qual_filters ? md_tagged : md_notags
// ojs_define() wraps a scalar from R in a length-1 array on some Quarto
// versions and passes it through on others, so unwrap defensively: the
// denominator being quietly NaN is exactly the failure this line exists to
// prevent.
n_all = Array.isArray(n_corpus) ? +n_corpus[0] : +n_corpus

Code

ns = Inputs.text().classList[0]

// custom css to override some ojs defaults for inputs
html`<style>

  .${ns} {
    --label-width: 80px;
  }

  form.${ns} {
    flex-wrap: wrap;
  }
  
  .${ns} div label {
    background-color: #f4f4f4;
    padding: 0.25rem 0.5rem;
    border-radius: 0.5rem;
    margin-right: 0.25rem;
    margin-bottom: 0.25rem;
    width: auto;
  }
  
  .${ns} div label:hover,
  .${ns} div label:active,
  .${ns} div label:focus {
    background-color: #cdecff;
  }
  
  .${ns} div input[type="number"] {
    background-color: #f4f4f4;
    padding: 0.25rem 0.5rem;
    border-radius: 0.5rem;
    flex-shrink: 3;
    border: none;
  }
  
  .${ns} select {
    background-color: #f4f4f4;
    border: none;
    border-radius: 0.5rem;
    padding: 0.25rem 0.5rem;
    //width: auto;
  }
  
  .${ns} .hist {
    width: 100%;
    display: flex;
    flex-direction: column;
    row-gap: 0em;
  }

</style>`

Code

_vi = ({
  "n_responses":               { label: "Number responses (n_responses)",                        base: 10 },
  "n_categories":              { label: "Number categories (n_categories)",                      base:  2 },
  "n_participants":            { label: "Number participants (n_participants)",                  base: 10 },
  "n_items":                   { label: "Number items (n_items)",                                base: 10 },
  "responses_per_participant": { label: "Responses per participant (responses_per_participant)", base: 10 },
  "responses_per_item":        { label: "Responses per item (responses_per_item)",               base: 10 },
  "density":                   { label: "Density (n_responses/(n_participants*n_items))",        base: 10 }
})

// compute and add in ranges for numeric variables
vi = Object.fromEntries(Object.entries(_vi).map(([k, v]) => [k, {...v, range: log_range(md.map(d => d[k]), v.base)}]))

// info for tag variables
ti = ({
  "longitudinal":                          { label: "Longitudinal"                          } ,
  "has_item_text":                         { label: "Has item text?"                        } ,
  "age_range":                             { label: "Age range"                             } ,
  "child_age__for_child_focused_studies_": { label: "Child age (for child-focused studies)" } ,
  "construct_type":                        { label: "Construct type"                        } ,
  "sample":                                { label: "Sample"                                } ,
  "measurement_tool":                      { label: "Measurement tool"                      } ,
  "item_format":                           { label: "Item format"                           } ,
  "primary_language_s_":                   { label: "Primary language(s)"                   } ,
  "license":                               { label: "License"                               } ,
})

Code

pow_format = (v, b) => b === 10 ? d3.format("~s")(Math.pow(b, v)) : Plot.formatNumber("en-US")(Math.pow(b, v))
interval_format = (b) => ([start, end]) => `[${pow_format(start, b)}; ${pow_format(end, b)}]`

// any base log
log = (x, b) => Math.log(x) / Math.log(b)
// log-transformed extent of array of values
log_range = (x, b) => [Math.floor(log(d3.min(x), b)), Math.ceil(log(d3.max(x), b))]

// interval input for a given numeric variable
var_interval = (v) => {
  //return interval(log_range(md.map(d => d[v]), vi[v].base), {
  return interval(vi[v].range, {
    step: 1, label: vi[v].label, format: interval_format(vi[v].base), width: "95%"
  })
}

// checkbox input for a given tag variable
tag_checkbox = (t) => Inputs.checkbox(tags[t], {value: tags[t]})

Code

viewof n_responses_range = var_interval("n_responses")
viewof n_categories_range = var_interval("n_categories")
viewof n_participants_range = var_interval("n_participants")
viewof n_items_range = var_interval("n_items")
viewof responses_per_participant_range = var_interval("responses_per_participant")
viewof responses_per_item_range = var_interval("responses_per_item")
viewof density_range = var_interval("density")
viewof variable = yamultiselect(vars.variable, { label: "Variable" })
viewof prefix = yamultiselect(vars.prefix, { label: "Variable prefix" })
// Collections (issue #1633). yamultiselect rather than the checkbox idiom used
// for the tag facets: 22 values is too many to scan, and the checkbox default
// is all-selected, which would read as a constraint the user never set. Empty
// here correctly means "no constraint" -- hence overlap_null_filter below.
// Lives in this panel, not the qualitative one, because the design collections
// come from metadata and so apply even when qualitative filtering is off.
viewof collection_sel = yamultiselect(collection_opts, { label: "Collection" })
range_filter = (v, v_range) => {
  return (d) => d[v] >= Math.pow(vi[v].base, v_range[0]) &
                d[v] <= Math.pow(vi[v].base, v_range[1])
}

// generate overlap filter function for a given categorical variable and input values
overlap_filter = (t, t_vals) => {
  return (d) => t_vals.some(v => d[t].includes(v))
}

// generate overlap filter function as above but that defaults to including everything if input values empty
overlap_null_filter = (t, t_vals) => {
  return (d) => t_vals.length ? t_vals.some(v => d[t].includes(v)) : true
}

// specify range-based inputs (as variable, values)
range_inputs = ([
  { v: "n_responses",               vals: n_responses_range               },
  { v: "n_categories",              vals: n_categories_range              },
  { v: "n_participants",            vals: n_participants_range            },
  { v: "n_items",                   vals: n_items_range                   },
  { v: "responses_per_participant", vals: responses_per_participant_range },
  { v: "responses_per_item",        vals: responses_per_item_range        },
  { v: "density",                   vals: density_range                   },
])

// specify overlap-based inputs (as variable, values)
overlap_inputs = ([
  { v: "longitudinal",                          vals: longitudinal_vals     },
  { v: "has_item_text",                         vals: has_item_text_vals    },
  { v: "age_range",                             vals: age_range_vals        },
  { v: "child_age__for_child_focused_studies_", vals: child_age_vals        },
  { v: "construct_type",                        vals: construct_type_vals   },
  { v: "sample",                                vals: sample_vals           },
  { v: "measurement_tool",                      vals: measurement_tool_vals },
  { v: "item_format",                           vals: item_format_vals      },
  { v: "primary_language_s_",                   vals: primary_language_vals },
  { v: "license",                               vals: license_vals          },
])

// specify variable and prefix filters
var_inputs = ([
  { v: "variable", vals: variable },
  { v: "prefix",   vals: prefix   },
])

// The multiselect shows "rct  (178)" so the size and the coverage caveat are
// visible at the point of choosing; map the chosen labels back to bare slugs.
collection_vals = collection_sel.map(o => collection_key[collection_opts.indexOf(o)])

// Null-filter semantics: nothing selected means no constraint.
//
// This does NOT reuse overlap_null_filter. That helper does `d[t].includes(v)`,
// which is exact membership on an array but SUBSTRING matching on a string --
// and ojs_define collapses a length-1 list to a bare string, so a table in
// exactly one collection arrives as e.g. "intensive_longitudinal". Substring
// matching would then make it match a "longitudinal" filter, because
// "intensive_longitudinal".includes("longitudinal") is true. Normalise to an
// array first and compare exactly.
as_array = (x) => Array.isArray(x) ? x : (x === null || x === undefined || x === "" ? [] : [x])
collection_filter = (vals) => (d) =>
  vals.length ? as_array(d.collection).some(c => vals.includes(c)) : true

// generate filter function for each input and combine them
range_filters = range_inputs.map(({ v, vals }) => range_filter(v, vals))
overlap_filters = use_qual_filters ? overlap_inputs.map(({ v, vals }) => overlap_filter(v, vals)) : []
overlap_null_filters = var_inputs.map(({ v, vals }) => overlap_null_filter(v, vals))
collection_filters = [collection_filter(collection_vals)]
all_filters = [...range_filters, ...overlap_filters, ...overlap_null_filters, ...collection_filters]

// apply filters to metadata
ds = all_filters.reduce((mdf, fun) => mdf.filter(fun), md)
// replace "NA" with null in filtered data
dsf = ds.map(d => Object.fromEntries(Object.entries(d).map(([key, value]) => [key, value === "NA" ? null : value])))
num_vars = new Map(Object.entries(vi).map(([key, value]) => [value.label, key]))
viewof x_var = Inputs.select(num_vars, {value: "n_items", label: "X axis"})
viewof y_var = Inputs.select(num_vars, {value: "n_participants", label: "Y axis"})
cat_array = use_qual_filters ? color_vars.map((t) => [ti[t].label, t]) : []
cat_vars = new Map([["None", null], ...cat_array])
viewof color_var = Inputs.select(cat_vars, {label: "Color"})
scheme = tol.QualMuted

// default color when there's no color variable
default_color = "darkgrey"

// color for missing values
unk_color = "lightgrey"

// indicator whether to use color
use_color = color_var !== null

// turn value into array if it isn't one
arrayify = (x) => Array.isArray(x) ? x : [x]

// given array of strings or single string, collapses to one comma separated string
stringify = (x) => x === null ? null : arrayify(x).join(", ")

// domain of values to use for color
color_vals = color_var === null ? default_color : tags[color_var].filter((v) => v !== "NA")

default_symbol = "circle"
select_symbol = "times"

default_size = 1
select_size = 4

// scatter plot
Plot.plot({
  x: {type: "log", base: vi[x_var].base},
  y: {type: "log", base: vi[y_var].base},
  width: 700,
  grid: true,
  color: {
    domain: color_vals,
    range: scheme,
    unknown: unk_color,
    legend: use_color
  },
  marks: [
    // points
    Plot.dot(dsf, {
      x: x_var,
      y: y_var,
      stroke: use_color ? (d) => stringify(d[color_var]) : default_color,
      symbol: (d) => row === null ? default_symbol : d.table !== row.table ? default_symbol : select_symbol,
      r: (d) => row === null ? default_size : d.table !== row.table ? default_size : select_size
    }),
    // regression line
    Plot.linearRegressionY(dsf, {
      x: x_var,
      y: y_var,
      stroke: use_color ? (d) => stringify(d[color_var]) : default_color
    }),
    // tooltips
    Plot.tip(dsf, Plot.pointer({
      x: x_var,
      y: y_var,
      stroke: use_color ? (d) => stringify(d[color_var]) : default_color,
      title: "table",
    }))
  ]
})
html`<i>Filtered to <b>${ds.length.toLocaleString("en")}</b> datasets.${
  Number.isFinite(n_all) && md.length < n_all
    ? html` This tool can show ${md.length.toLocaleString("en")} of the <b>${n_all.toLocaleString("en")}</b> tables in the corpus; the ${(n_all - md.length).toLocaleString("en")} it cannot show ${use_qual_filters
        ? "are missing tags, a bibliography entry, or quantitative descriptors \u2014 turn off qualitative filtering below to include the untagged ones"
        : "are missing a bibliography entry or quantitative descriptors"}.`
    : html` ${md.length.toLocaleString("en")} datasets are available to filter.`
}</i>`
sparkbar = (max, b) => {
  return x => htl.html`<div class="sparkbar" style="width: ${100 * log(x, b) / log(max, b)}%;">${x.toLocaleString("en")}`
}

// sparkbar element generator for a given numeric variable
var_sparkbar = (v) => sparkbar(d3.max(ds, d => d[v]), vi[v].base)

// sparkbars for each numeric variable
sparks = Object.fromEntries(Object.entries(vi).map(([key, value]) => [key, var_sparkbar(key)]))

// inline histogram element given array of values and header string
inlinehist = (vals, head) => {
  return htl.html`<span class="hist">${hist(vals)}${head}</span>`
}

// inline histogram for a given numeric variable
var_hist = (v) => inlinehist(ds.map(d => log(d[v], vi[v].base)), v)

// histograms for each numeric variable
hists = Object.fromEntries(Object.entries(vi).map(([key, value]) => [key, var_hist(key)]))

// combine two arrays into pairs in one array
zip = (a, b) => a.map((k, i) => [k, b[i]])

// set colors for array of values, using default if there are too values
pal = (v, colors, default_color) => v.length <= colors.length ? colors.slice(0, v.length) : Array(v.length).fill(default_color)

// create object with colors for each tag value
tag_colors = Object.fromEntries(Object.entries(tags).map(([tag, tag_values]) => {
  //const vals = tag_values.sort().filter((v) => v !== "NA");
  const vals = tag_values.filter((v) => v !== "NA");
  return [tag, Object.fromEntries(zip(vals, pal(vals, scheme, default_color)))]
}))

// badge element for a given tag and values
badgify = (vals, tag) => htl.html`<div>
  ${arrayify(vals).map(val => htl.html.fragment`
    <span class="badge" style="background-color: ${tag_colors[tag][val]};">${val}</span>`)}
</div>`

// badge elements for each tag
badges = Object.fromEntries(Object.keys(ti).map(tag => [tag, d => badgify(d, tag)]))
// data table for selected tables
viewof row = Inputs.table(dsf, {
  rows: 20,
  sort: "n_responses", reverse: true,
  required: false, multiple: false,
  //value: dsf[1],
  // show columns "table", all numeric variables, and (if enabled) all tag variables
  columns: ["table", ...Array.from(Object.keys(vi)), ...(use_qual_filters ? Array.from(Object.keys(ti)) : [])],
  // use inline histograms in header
  header: hists,
  // use sparkbars and badges in cells
  format: {...sparks, ...badges}
})
has_page = new Set(
  (typeof page_tables === "undefined" || page_tables === null
     ? []
     : (Array.isArray(page_tables) ? page_tables : [page_tables])
  ).map(s => String(s).toLowerCase())
)

table_page_href = (name) => `/tables/${String(name).toLowerCase()}/`
info = ({
  "table":       { label: "Dataset"        },
  "description": { label: "Description"    },
  "variables":   { label: "Variables"      },
  "url":         { label: "Link to table"  },
  "reference":   { label: "Reference"      },
  "data_url":    { label: "Link to source" },
  "license":     { label: "License"        },
})

// element for each info entry ($head: $text) with $text wrapped in <a> if head matches "Link"
// and $text is a URL. data_url also holds values like "NA" and "Author Permission";
// linked, those became relative hrefs that crawlers followed to 404s.
pullout = (head, text) => html.fragment`<p><b>${head}</b>: ${/Link/.test(head) && /^https?:\/\//.test(String(text)) ? html.fragment`<a href="${text}" target="_blank">${text}</a>` : text}</p>`

// elements for all info entries, placeholder if no table selected
placeholder = "No table selected"
row ? html`<div class="pullout">
  ${Object.entries(info).map(([field, properties]) => pullout(properties.label, row[field]))}
  ${has_page.has(String(row.table).toLowerCase())
      ? html.fragment`<p><b>Citable page</b>: <a href="${table_page_href(row.table)}">${table_page_href(row.table)}</a> &mdash; a permanent page for this table, naming the IRW version it describes.</p>`
      : html.fragment``}
</div>` : html`<div class="pullout"><em>${placeholder}</em></div>`
fillTemplate = (template, values) => {
  return template.replace(/\{\{(\w+)\}\}/g, (match, key) => {
    return values[key] !== undefined ? values[key] : match;
  });
};

// given array, create string representation R character vector of values
c = (vals) => `c(${vals.map(s => `"${s}"`).join(", ")})`

// construct arguments for range inputs if they're not their default values
is_default_range = (v, vals) => vals[0] === vi[v].range[0] && vals[1] === vi[v].range[1]
range_arg = (v, vals) => is_default_range(v, vals) ? null : `${v} = c(${Math.pow(vi[v].base, vals[0])}, ${Math.pow(vi[v].base, vals[1])})`
range_args = range_inputs.map(({ v, vals }) => range_arg(v, vals))

// construct arguments for overlap inputs if they're not their default values
// (only meaningful when qualitative filtering is enabled)
is_default_overlap = (v, vals) => tags[v].every(t => vals.includes(t))
overlap_arg = (v, vals) => is_default_overlap(v, vals) ? null : `${v} = ${c(vals.filter(e => e !== "NA"))}`
overlap_args = use_qual_filters ? overlap_inputs.filter(d => d.v !== l && d.v !== "has_item_text").map(({ v, vals }) => overlap_arg(v, vals)) : []

// special handling to construct var argument by combining variables and prefixes
var_arg_vals = var_inputs.map(({v, vals}) => v === "prefix" ? vals.map(s => `${s}_`) : vals).flat()
var_arg = var_arg_vals.length ? `var = ${c(var_arg_vals)}` : null

// special handling to construct longitudinal argument as boolean
l = "longitudinal"
long_vals = overlap_inputs.filter(d => d.v === l)[0].vals
long_arg = use_qual_filters && !is_default_overlap(l, long_vals) ? `${l} = ${long_vals[0] === "longitudinal" ? "TRUE" : "FALSE"}` : null

// collection argument; the OJS name must match irw_filter()'s formal exactly
collection_arg = collection_vals.length ? `collection = ${c(collection_vals)}` : null

// combine all arguments
filter_args = [...range_args, var_arg, collection_arg, ...overlap_args, long_arg].filter(e => e !== null)
filter_str = filter_args.length ? `\n  ${filter_args.join(",\n  ")}  \n` : ""

// load template and fill arguments into it
r_filter_template = FileAttachment("resources/templates/query_filter_snippet.R").text();
r_filter_snippet = () => fillTemplate(r_filter_template, { filter_str: filter_str })
// code snippet box with copy button
ds.length === 0 ? html`<em>No datasets match filters.</em>` : html`
<div class="pullout">
<div class="code-copy-outer-scaffold">
<div class="sourceCode">
<pre class="sourceCode r code-with-copy">
<code class="sourceCode r">
<span>${r_filter_snippet()}</span></code></pre>
</div>
<button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button>
</div>
</div>
`
viewof use_qual_filters = Inputs.toggle({label: "Enable qualitative filtering", value: true})
html`<style>
#qual-filter-panel { opacity: ${use_qual_filters ? 1 : 0.45}; pointer-events: ${use_qual_filters ? "auto" : "none"}; transition: opacity 0.15s ease; }
</style>`
html`${use_qual_filters ? "" : html`<p><em>Qualitative filtering is disabled. Datasets missing qualitative tags (age range, sample, license, etc.) are now included in the results, but can't be filtered on those tags. Enable the toggle above to filter by them (this will also exclude untagged datasets again).</em></p>`}`
viewof longitudinal_vals = tag_checkbox("longitudinal")
viewof has_item_text_vals = tag_checkbox("has_item_text")
viewof age_range_vals = tag_checkbox("age_range")
viewof child_age_vals = tag_checkbox("child_age__for_child_focused_studies_")
viewof construct_type_vals = tag_checkbox("construct_type")
viewof sample_vals = tag_checkbox("sample")
viewof measurement_tool_vals = tag_checkbox("measurement_tool")
viewof item_format_vals = tag_checkbox("item_format")
viewof primary_language_vals = tag_checkbox("primary_language_s_")
viewof license_vals = tag_checkbox("license")

Filter by quantitative properties.

Filter by variables.

Information on selected dataset

Code snippet

The R code snippet below uses the irw package to fetch all of the tables that match the filters specified here. Note: a difference between the functionality of the filters here and the behavior of irw_filter() is that specifying values for a given argument of irw_filter() will result in NA values being dropped, while they can be included here. The “Has item text?” filter is also not reflected in the snippet, since irw_filter() does not support filtering by item-text availability; use irw_list_itemtext_tables() separately for that. When qualitative filtering is disabled (see the toggle above the “Filter by qualitative features” panel), none of the qualitative-tag arguments (age range, sample, license, etc.) are included in the snippet, matching the fact that no qualitative filtering is applied to the table above.

Filter by qualitative features.

Look up a single dataset

Below we show an example dataset and illustrations of how to access the data programmatically. You can also explore the data here. The search covers every data family: tables from the competition, nominal, conjoint, rankings and simulated families are listed with their family after the name, and the citation hint carries the source the packages need to fetch them.

Individual tables also have their own citable pages, listed at IRW table pages and linked from the panel above where one exists.

Code

dictionary = data_index

function createDatasetMap(df) {
  let dataArray = Array.from({ length: df[Object.keys(df)[0]].length }, (_, i) => {
    return Object.fromEntries(Object.entries(df).map(([key, values]) => [key, values[i]]));
  });

  // Sort the array alphabetically by dataset
  dataArray.sort((a, b) => a.dataset.localeCompare(b.dataset));

  // Create the map from sorted array
  let datasetMap = new Map();
  for (let row of dataArray) {
    datasetMap.set(row.dataset, row);
  }
  return datasetMap;
}

// Create dataset map from data_index instead of metadata
dataset_map = createDatasetMap(data_index);

viewof dataset_name = {
  const allNames = Array.from(dataset_map.keys());
  let selectedName = allNames[0];
  const familyOf = (name) => source_labels[dataset_map.get(name)?.source] ?? "";

  // Attach list to body so it's never clipped by layout overflow
  const list = document.createElement("div");
  list.style.cssText = "position:fixed;z-index:99999;max-height:260px;overflow-y:auto;border:1px solid #ccc;border-radius:0 0 4px 4px;background:white;display:none;box-shadow:0 4px 12px rgba(0,0,0,0.15);";
  document.body.appendChild(list);

  const container = html`<div style="display:inline-block;width:400px;margin-bottom:20px;">
    <label style="font-weight:600;display:block;margin-bottom:4px;">Search Dataset</label>
    <div style="position:relative;">
      <svg style="position:absolute;left:8px;top:50%;transform:translateY(-50%);pointer-events:none;color:#888;" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
        <circle cx="11" cy="11" r="8"/><line x1="21" y1="21" x2="16.65" y2="16.65"/>
      </svg>
      <input type="text" placeholder="Type to search datasets…"
        style="width:100%;padding:6px 10px 6px 28px;font-size:smaller;border:1px solid #ccc;border-radius:4px;box-sizing:border-box;" />
    </div>
  </div>`;

  const input = container.querySelector("input");

  function positionList() {
    const r = input.getBoundingClientRect();
    list.style.top = r.bottom + "px";
    list.style.left = r.left + "px";
    list.style.width = r.width + "px";
  }

  function renderList(filter) {
    const matches = filter
      ? allNames.filter(n => n.toLowerCase().includes(filter.toLowerCase()))
      : allNames;
    list.innerHTML = "";
    for (const name of matches.slice(0, 300)) {
      const item = document.createElement("div");
      item.textContent = name;
      const fam = familyOf(name);
      if (fam) {
        const tag = document.createElement("span");
        tag.textContent = fam;
        tag.style.cssText = "float:right;color:#888;font-size:11px;";
        item.appendChild(tag);
      }
      item.style.cssText = "padding:5px 10px;cursor:pointer;font-size:13px;border-bottom:1px solid #f0f0f0;";
      item.onmousedown = (e) => {
        e.preventDefault();
        selectedName = name;
        input.value = name;
        list.style.display = "none";
        container.value = selectedName;
        container.dispatchEvent(new Event("input", {bubbles: true}));
      };
      item.onmouseenter = () => item.style.backgroundColor = "#f0f4ff";
      item.onmouseleave = () => item.style.backgroundColor = "";
      list.appendChild(item);
    }
  }

  input.addEventListener("focus", () => {
    input.value = "";
    renderList("");
    positionList();
    list.style.display = "block";
  });
  input.addEventListener("input", (e) => {
    e.stopPropagation();
    renderList(input.value);
    positionList();
    list.style.display = "block";
  });
  input.addEventListener("blur", () => {
    setTimeout(() => { list.style.display = "none"; input.value = selectedName; }, 150);
  });

  function onScroll(e) { if (list.contains(e.target)) return; list.style.display = "none"; input.value = selectedName; }
  window.addEventListener("scroll", onScroll, true);

  invalidation.then(() => { list.remove(); window.removeEventListener("scroll", onScroll, true); });

  input.value = selectedName;
  container.value = selectedName;
  return container;
}

// Function to find dataset info from dictionary (fixed field name)
function findDatasetInfo(dictionary, datasetName) {
  // Convert dictionary column data into array of objects
  const dictionaryArray = Array.from({ length: dictionary[Object.keys(dictionary)[0]].length }, (_, i) => {
    return Object.fromEntries(Object.entries(dictionary).map(([key, values]) => [key, values[i]]));
  });

  // Now we can use find on the array
  const dictRow = dictionaryArray.find(row => row.dataset === datasetName);
  return {
    reference: dictRow?.reference || "Reference not available",
    BibTex: dictRow?.BibTex || "BibTeX not available",
    doi: dictRow?.doi,
    source: dictRow?.source || "core"
  };
}

dataset_info = findDatasetInfo(dictionary, dataset_name);

// The family each `source` code names, for the dropdown and the hint. Core
// tables carry no label: they are the default everywhere.
source_labels = ({comp: "competitions", nom: "nominal", conj: "conjoint", rank: "rankings", sim: "simulated"})

// Create the display elements with reference and BibTeX
display = {
  const sanitizedDatasetName = dataset_name.replace(/\./g, "_"); // Replace all dots with underscores
  const dsRefIdx = table_ds_ref.table.findIndex(t => t === dataset_name.toLowerCase());
  const dsRef = dsRefIdx >= 0 ? table_ds_ref.ds_ref[dsRefIdx] : "item_response_warehouse:as2e";
  const url = `https://redivis.com/embed/tables/datapages.${dsRef}:current.${sanitizedDatasetName}`;

  const container = html`<div>
    <div style="margin-bottom: 20px;">
      <button onclick="toggleInfo(this)" style="
        padding: 8px 16px;
        background-color: #4287f5;
        color: white;
        border: none;
        border-radius: 6px;
        cursor: pointer;
        font-weight: 500;
        display: flex;
        align-items: center;
        gap: 5px;
        transition: background-color 0.2s;
        box-shadow: 0 2px 4px rgba(0,0,0,0.1);
      ">
        <span>Show Citation Info</span>
        <svg width="12" height="12" viewBox="0 0 12 12" fill="none" style="transform: rotate(0deg); transition: transform 0.3s;">
          <path d="M2 4L6 8L10 4" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
        </svg>
      </button>

      <div style="
        display: none;
        margin-top: 10px;
        padding: 20px;
        background-color: #ffffff;
        border-radius: 8px;
        border: 1px solid #e1e4e8;
        transition: all 0.3s ease;
        box-shadow: 0 2px 8px rgba(0,0,0,0.05);
      ">
        <div style="display: flex; gap: 20px;">
          <div style="flex: 1;">
            <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 5px;">
              <div style="font-weight: 600; color: #24292e;">Reference</div>
              <button onclick="copyText(this, 'reference')" style="
                padding: 4px 12px;
                background-color: #f6f8fa;
                border: 1px solid #e1e4e8;
                border-radius: 6px;
                cursor: pointer;
                font-size: 0.8em;
                color: #24292e;
                display: flex;
                align-items: center;
                gap: 4px;
                transition: all 0.2s;
              ">
                <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor">
                  <path d="M8 4v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V7.242a2 2 0 0 0-.602-1.43L16.083 2.57A2 2 0 0 0 14.685 2H10a2 2 0 0 0-2 2z" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
                  <path d="M16 18v2a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V9a2 2 0 0 1 2-2h2" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
                </svg>
                Copy
              </button>
            </div>
            <div style="
              font-size: 0.95em;
              line-height: 1.6;
              color: #24292e;
              font-style: italic;
              background-color: #f8f9fa;
              padding: 12px;
              border-radius: 6px;
              border: 1px solid #e1e4e8;
            " class="reference-text">${dataset_info.reference}</div>
          </div>
          <div style="flex: 1;">
            <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 5px;">
              <div style="font-weight: 600; color: #24292e;">BibTeX</div>
              <button onclick="copyText(this, 'bibtex')" style="
                padding: 4px 12px;
                background-color: #f6f8fa;
                border: 1px solid #e1e4e8;
                border-radius: 6px;
                cursor: pointer;
                font-size: 0.8em;
                color: #24292e;
                display: flex;
                align-items: center;
                gap: 4px;
                transition: all 0.2s;
              ">
                <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor">
                  <path d="M8 4v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V7.242a2 2 0 0 0-.602-1.43L16.083 2.57A2 2 0 0 0 14.685 2H10a2 2 0 0 0-2 2z" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
                  <path d="M16 18v2a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V9a2 2 0 0 1 2-2h2" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
                </svg>
                Copy
              </button>
            </div>
            <pre style="
              margin: 0;
              padding: 12px;
              background-color: #f8f9fa;
              border-radius: 6px;
              border: 1px solid #e1e4e8;
              font-size: 0.9em;
              line-height: 1.5;
              overflow-x: auto;
              white-space: pre-wrap;
              word-wrap: break-word;
              color: #24292e;
              font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Consolas', 'source-code-pro', monospace;
            " class="bibtex-text"></pre>
          </div>
        </div>

        <!-- R Package Citation -->
        <div style="margin-top: 20px;">
          <button onclick="toggleRCitation(this)" style="
            padding: 6px 12px;
            background-color: #4287f5;
            color: white;
            border: none;
            border-radius: 6px;
            cursor: pointer;
            font-size: 0.9em;
            transition: background-color 0.2s;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
          ">To cite using the IRW packages</button>

          <div class="r-citation" style="
            display: none;
            margin-top: 10px;
            padding: 12px;
            background-color: #f8f9fa;
            border-radius: 6px;
            border: 1px solid #e1e4e8;
            font-family: monospace;
          ">
            <div style="
              padding: 8px;
              background-color: #f8f9fa;
              border-left: 3px solid #4287f5;
              margin-bottom: 10px;
            ">
              R: <code>irw_save_bibtex("${dataset_name}", "refs.bib"${dataset_info.source === "core" ? "" : `, source = "${dataset_info.source}"`})</code><br>
              Python: <code>irw.save_bibtex("${dataset_name}", "refs.bib"${dataset_info.source === "core" ? "" : `, source="${dataset_info.source}"`})</code>
            </div>
          </div>
        </div>
      </div>
    </div>
    <iframe id="myIframe" width="800" height="500" allowfullscreen style="border: 1px solid #e1e4e8; border-radius: 8px;" src="${url}"></iframe>
  </div>`;

  // Add toggle functions to window object
  window.toggleInfo = function(button) {
    const infoDiv = button.nextElementSibling;
    const arrow = button.querySelector('svg');
    const isVisible = infoDiv.style.display === 'block';

    infoDiv.style.display = isVisible ? 'none' : 'block';
    arrow.style.transform = isVisible ? 'rotate(0deg)' : 'rotate(180deg)';
    button.querySelector('span').textContent = isVisible ? 'Show Citation Info' : 'Hide Citation Info';
  };

  window.toggleRCitation = function(button) {
    const citationDiv = button.nextElementSibling;
    citationDiv.style.display = citationDiv.style.display === 'none' ? 'block' : 'none';
  };

  // Add copy function to window object
  window.copyText = function(button, type) {
    const container = button.closest('div').parentElement;
    const text = type === 'reference' ?
      container.querySelector('.reference-text').textContent :
      container.querySelector('.bibtex-text').textContent;

    navigator.clipboard.writeText(text).then(() => {
      const originalText = button.textContent;
      button.textContent = 'Copied!';
      button.style.backgroundColor = '#28a745';
      button.style.borderColor = '#28a745';
      button.style.color = 'white';

      setTimeout(() => {
        button.innerHTML = `
          <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor">
            <path d="M8 4v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V7.242a2 2 0 0 0-.602-1.43L16.083 2.57A2 2 0 0 0 14.685 2H10a2 2 0 0 0-2 2z" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
            <path d="M16 18v2a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V9a2 2 0 0 1 2-2h2" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
          </svg>
          Copy
        `;
        button.style.backgroundColor = '#f6f8fa';
        button.style.borderColor = '#e1e4e8';
        button.style.color = '#24292e';
      }, 2000);
    });
  };

// Update BibTeX text
const bibtexElement = container.querySelector('.bibtex-text');
if (bibtexElement) {
  bibtexElement.textContent = dataset_info.BibTex || "BibTeX not available";
}

  return container;
}

If you have any questions or feedback, please feel free to contact us.