Refactor topo viz to do as much computation upfront & use faster data structures

Now all data loading happens in the TopoViz component as well as
computation of resource proportions.

Allocation selection state is also managed centrally uses a dedicated
structure indexed by group key (job id and task group name). This way
allocations don't need to be scanned at the node level, which is O(n) at
the best (assuming no ember overhead on recomputes).

ui/app/components/topo-viz.js

@@ -1,7 +1,8 @@
 import Component from '@glimmer/component';
 import { tracked } from '@glimmer/tracking';
-import { action } from '@ember/object';
+import { action, set } from '@ember/object';
 import { run } from '@ember/runloop';
+import { task } from 'ember-concurrency';
 import { scaleLinear } from 'd3-scale';
 import { extent } from 'd3-array';
 import RSVP from 'rsvp';
@@ -10,6 +11,7 @@ export default class TopoViz extends Component {
   @tracked heightScale = null;
   @tracked isLoaded = false;
   @tracked element = null;
+  @tracked topology = {};
 
   @tracked activeAllocation = null;
   @tracked activeEdges = [];
@@ -22,17 +24,88 @@ export default class TopoViz extends Component {
     return this.activeAllocation && this.activeAllocation.belongsTo('job').id();
   }
 
-  get datacenters() {
+  dataForNode(node) {
-    const datacentersMap = this.args.nodes.reduce((datacenters, node) => {
+    return {
-      if (!datacenters[node.datacenter]) datacenters[node.datacenter] = [];
+      node,
-      datacenters[node.datacenter].push(node);
+      datacenter: node.datacenter,
+      memory: node.resources.memory,
+      cpu: node.resources.cpu,
+      allocations: [],
+    };
+  }
+
+  dataForAllocation(allocation, node) {
+    const jobId = allocation.belongsTo('job').id();
+    return {
+      allocation,
+      node,
+      jobId,
+      groupKey: JSON.stringify([jobId, allocation.taskGroupName]),
+      memory: allocation.resources.memory,
+      cpu: allocation.resources.cpu,
+      memoryPercent: allocation.resources.memory / node.memory,
+      cpuPercent: allocation.resources.cpu / node.cpu,
+      isSelected: false,
+    };
+  }
+
+  @task(function*() {
+    const nodes = this.args.nodes;
+    const allocations = this.args.allocations;
+
+    // Nodes are probably partials and we'll need the resources on them
+    // TODO: this is an API update waiting to happen.
+    yield RSVP.all(nodes.map(node => (node.isPartial ? node.reload() : RSVP.resolve(node))));
+
+    // Wrap nodes in a topo viz specific data structure and build an index to speed up allocation assignment
+    const nodeContainers = [];
+    const nodeIndex = {};
+    nodes.forEach(node => {
+      const container = this.dataForNode(node);
+      nodeContainers.push(container);
+      nodeIndex[node.id] = container;
+    });
+
+    // Wrap allocations in a topo viz specific data structure, assign allocations to nodes, and build an allocation
+    // index keyed off of job and task group
+    const allocationIndex = {};
+    allocations.forEach(allocation => {
+      const nodeId = allocation.belongsTo('node').id();
+      const nodeContainer = nodeIndex[nodeId];
+      if (!nodeContainer)
+        throw new Error(`Node ${nodeId} for alloc ${allocation.id} not in index???`);
+
+      const allocationContainer = this.dataForAllocation(allocation, nodeContainer);
+      nodeContainer.allocations.push(allocationContainer);
+
+      const key = allocationContainer.groupKey;
+      if (!allocationIndex[key]) allocationIndex[key] = [];
+      allocationIndex[key].push(allocationContainer);
+    });
+
+    // Group nodes into datacenters
+    const datacentersMap = nodeContainers.reduce((datacenters, nodeContainer) => {
+      if (!datacenters[nodeContainer.datacenter]) datacenters[nodeContainer.datacenter] = [];
+      datacenters[nodeContainer.datacenter].push(nodeContainer);
       return datacenters;
     }, {});
 
-    return Object.keys(datacentersMap)
+    // Turn hash of datacenters into a sorted array
+    const datacenters = Object.keys(datacentersMap)
       .map(key => ({ name: key, nodes: datacentersMap[key] }))
       .sortBy('name');
-  }
+
+    const topology = {
+      datacenters,
+      allocationIndex,
+      selectedKey: null,
+      heightScale: scaleLinear()
+        .range([15, 40])
+        .domain(extent(nodeContainers.mapBy('memory'))),
+    };
+    this.topology = topology;
+  })
+  buildTopology;
 
   @action
   async loadNodes() {
@@ -81,11 +154,37 @@ export default class TopoViz extends Component {
     if (this.activeAllocation === allocation) {
       this.activeAllocation = null;
       this.activeEdges = [];
+
+      if (this.topology.selectedKey) {
+        const selectedAllocations = this.topology.allocationIndex[this.topology.selectedKey];
+        if (selectedAllocations) {
+          selectedAllocations.forEach(allocation => {
+            set(allocation, 'isSelected', false);
+          });
+        }
+        set(this.topology, 'selectedKey', null);
+      }
     } else {
       this.activeAllocation = allocation;
+      const selectedAllocations = this.topology.allocationIndex[this.topology.selectedKey];
+      if (selectedAllocations) {
+        selectedAllocations.forEach(allocation => {
+          set(allocation, 'isSelected', false);
+        });
+      }
+
+      set(this.topology, 'selectedKey', allocation.groupKey);
+      const newAllocations = this.topology.allocationIndex[this.topology.selectedKey];
+      if (newAllocations) {
+        newAllocations.forEach(allocation => {
+          set(allocation, 'isSelected', true);
+        });
+      }
+
       this.computedActiveEdges();
     }
-    if (this.args.onAllocationSelect) this.args.onAllocationSelect(this.activeAllocation);
+    if (this.args.onAllocationSelect)
+      this.args.onAllocationSelect(this.activeAllocation && this.activeAllocation.allocation);
   }
 
   @action
@@ -93,7 +192,7 @@ export default class TopoViz extends Component {
     // Wait a render cycle
     run.next(() => {
       const activeEl = this.element.querySelector(
-        `[data-allocation-id="${this.activeAllocation.id}"]`
+        `[data-allocation-id="${this.activeAllocation.allocation.id}"]`
       );
       const selectedAllocations = this.element.querySelectorAll('.memory .bar.is-selected');
       const activeBBox = activeEl.getBoundingClientRect();

ui/app/components/topo-viz/datacenter.js

@@ -1,19 +1,13 @@
-import RSVP from 'rsvp';
 import Component from '@glimmer/component';
 import { tracked } from '@glimmer/tracking';
 import { action } from '@ember/object';
 
-export default class TopoVizNode extends Component {
+export default class TopoVizDatacenter extends Component {
   @tracked scheduledAllocations = [];
   @tracked aggregatedNodeResources = { cpu: 0, memory: 0 };
-  @tracked isLoaded = false;
-
-  get aggregateNodeResources() {
-    return this.args.nodes.mapBy('resources');
-  }
 
   get aggregatedAllocationResources() {
-    return this.scheduledAllocations.mapBy('resources').reduce(
+    return this.scheduledAllocations.reduce(
       (totals, allocation) => {
         totals.cpu += allocation.cpu;
         totals.memory += allocation.memory;
@@ -24,15 +18,13 @@ export default class TopoVizNode extends Component {
   }
 
   @action
-  async loadAllocations() {
+  loadAllocations() {
-    await RSVP.all(this.args.nodes.mapBy('allocations'));
+    this.scheduledAllocations = this.args.datacenter.nodes.reduce(
-
+      (all, node) => all.concat(node.allocations.filterBy('allocation.isScheduled')),
-    this.scheduledAllocations = this.args.nodes.reduce(
-      (all, node) => all.concat(node.allocations.filterBy('isScheduled')),
       []
     );
 
-    this.aggregatedNodeResources = this.args.nodes.mapBy('resources').reduce(
+    this.aggregatedNodeResources = this.args.datacenter.nodes.reduce(
       (totals, node) => {
         totals.cpu += node.cpu;
         totals.memory += node.memory;
@@ -41,7 +33,6 @@ export default class TopoVizNode extends Component {
       { cpu: 0, memory: 0 }
     );
 
-    this.isLoaded = true;
     this.args.onLoad && this.args.onLoad();
   }
 }

ui/app/components/topo-viz/node.js

@@ -1,6 +1,6 @@
 import Component from '@glimmer/component';
 import { tracked } from '@glimmer/tracking';
-import { action, set } from '@ember/object';
+import { action } from '@ember/object';
 import { guidFor } from '@ember/object/internals';
 
 export default class TopoVizNode extends Component {
@@ -10,7 +10,7 @@ export default class TopoVizNode extends Component {
   @tracked activeAllocation = null;
 
   get height() {
-    return this.args.heightScale ? this.args.heightScale(this.args.node.resources.memory) : 15;
+    return this.args.heightScale ? this.args.heightScale(this.args.node.memory) : 15;
   }
 
   get labelHeight() {
@@ -51,19 +51,15 @@ export default class TopoVizNode extends Component {
   }
 
   get count() {
-    return this.args.node.get('allocations.length');
+    return this.args.node.allocations.length;
   }
 
   get allocations() {
-    // return this.args.node.allocations.filterBy('isScheduled').sortBy('resources.memory');
-    const totalCPU = this.args.node.resources.cpu;
-    const totalMemory = this.args.node.resources.memory;
-
     // Sort by the delta between memory and cpu percent. This creates the least amount of
     // drift between the positional alignment of an alloc's cpu and memory representations.
-    return this.args.node.allocations.filterBy('isScheduled').sort((a, b) => {
+    return this.args.node.allocations.filterBy('allocation.isScheduled').sort((a, b) => {
-      const deltaA = Math.abs(a.resources.memory / totalMemory - a.resources.cpu / totalCPU);
+      const deltaA = Math.abs(a.memoryPercent - a.cpuPercent);
-      const deltaB = Math.abs(b.resources.memory / totalMemory - b.resources.cpu / totalCPU);
+      const deltaB = Math.abs(b.memoryPercent - b.cpuPercent);
       return deltaA - deltaB;
     });
   }
@@ -89,8 +85,6 @@ export default class TopoVizNode extends Component {
     if (newWidth !== this.dimensionsWidth) {
       this.dimensionsWidth = newWidth;
       this.data = this.computeData(this.dimensionsWidth);
-    } else {
-      this.data = this.setSelection();
     }
   }
 
@@ -110,51 +104,23 @@ export default class TopoVizNode extends Component {
   }
 
   containsActiveTaskGroup() {
-    return this.allocations.some(
+    return this.args.node.allocations.some(
       allocation =>
         allocation.taskGroupName === this.args.activeTaskGroup &&
         allocation.belongsTo('job').id() === this.args.activeJobId
     );
   }
 
-  setSelection() {
-    this.data.cpu.forEach(cpu => {
-      set(
-        cpu,
-        'isSelected',
-        cpu.allocation.taskGroupName === this.args.activeTaskGroup &&
-          cpu.allocation.belongsTo('job').id() === this.args.activeJobId
-      );
-    });
-    this.data.memory.forEach(memory => {
-      set(
-        memory,
-        'isSelected',
-        memory.allocation.taskGroupName === this.args.activeTaskGroup &&
-          memory.allocation.belongsTo('job').id() === this.args.activeJobId
-      );
-    });
-    return this.data;
-  }
-
   computeData(width) {
-    // TODO: differentiate reserved and resources
+    const allocations = this.allocations;
-    if (!this.args.node.resources) return;
-
-    const totalCPU = this.args.node.resources.cpu;
-    const totalMemory = this.args.node.resources.memory;
     let cpuOffset = 0;
     let memoryOffset = 0;
 
     const cpu = [];
     const memory = [];
-    for (const allocation of this.allocations) {
+    for (const allocation of allocations) {
-      const cpuPercent = allocation.resources.cpu / totalCPU;
+      const { cpuPercent, memoryPercent, isSelected } = allocation;
-      const memoryPercent = allocation.resources.memory / totalMemory;
+      const isFirst = allocation === allocations[0];
-      const isFirst = allocation === this.allocations[0];
-      const isSelected =
-        allocation.taskGroupName === this.args.activeTaskGroup &&
-        allocation.belongsTo('job').id() === this.args.activeJobId;
 
       let cpuWidth = cpuPercent * width - 1;
       let memoryWidth = memoryPercent * width - 1;
@@ -169,21 +135,19 @@ export default class TopoVizNode extends Component {
 
       cpu.push({
         allocation,
-        isSelected,
         offset: cpuOffset * 100,
         percent: cpuPercent * 100,
         width: cpuWidth,
         x: cpuOffset * width + (isFirst ? 0 : 0.5) + (isSelected ? 0.5 : 0),
-        className: allocation.clientStatus,
+        className: allocation.allocation.clientStatus,
       });
       memory.push({
         allocation,
-        isSelected,
         offset: memoryOffset * 100,
         percent: memoryPercent * 100,
         width: memoryWidth,
         x: memoryOffset * width + (isFirst ? 0 : 0.5) + (isSelected ? 0.5 : 0),
-        className: allocation.clientStatus,
+        className: allocation.allocation.clientStatus,
       });
 
       cpuOffset += cpuPercent;
@@ -209,7 +173,3 @@ export default class TopoVizNode extends Component {
     };
   }
 }
-
-// capture width on did insert element
-// update width on window resize
-// recompute data when width changes

ui/app/templates/components/topo-viz.hbs

@@ -1,13 +1,12 @@
-<div class="topo-viz {{if (eq this.datacenters.length 1) "is-single-column"}}" {{did-insert this.loadNodes}} {{did-insert this.captureElement}}>
+<div class="topo-viz {{if (eq this.datacenters.length 1) "is-single-column"}}" {{did-insert (perform this.buildTopology)}} {{did-insert this.captureElement}}>
-  {{#if this.isLoaded}}
+  {{#if this.buildTopology.isRunning}}
-    {{#each this.datacenters as |dc|}}
+    <div class="has-text-centered"><LoadingSpinner /></div>
+  {{else}}
+    {{#each this.topology.datacenters as |dc|}}
       <TopoViz::Datacenter
-        @datacenter={{dc.name}}
+        @datacenter={{dc}}
-        @nodes={{dc.nodes}}
+        @heightScale={{this.topology.heightScale}}
-        @heightScale={{this.heightScale}}
         @onAllocationSelect={{this.associateAllocations}}
-        @activeTaskGroup={{this.activeTaskGroup}}
-        @activeJobId={{this.activeJobId}}
         @onLoad={{action this.masonry}}/>
     {{/each}}
 

ui/app/templates/components/topo-viz/datacenter.hbs

@@ -1,23 +1,19 @@
 <div class="boxed-section topo-viz-datacenter" {{did-insert this.loadAllocations}}>
   <div class="masonry-container">
     <div class="boxed-section-head is-hollow">
-      <strong>{{@datacenter}}</strong>
+      <strong>{{@datacenter.name}}</strong>
       <span class="bumper-left">{{this.scheduledAllocations.length}} Allocs</span>
-      <span class="bumper-left">{{@nodes.length}} Nodes</span>
+      <span class="bumper-left">{{@datacenter.nodes.length}} Nodes</span>
       <span class="bumper-left is-faded">{{this.aggregatedAllocationResources.memory}}/{{this.aggregatedNodeResources.memory}} MiB,
         {{this.aggregatedAllocationResources.cpu}}/{{this.aggregatedNodeResources.cpu}} Mhz</span>
     </div>
     <div class="boxed-section-body">
-      {{#if this.isLoaded}}
+      {{#each @datacenter.nodes as |node|}}
-        {{#each @nodes as |node|}}
+        <TopoViz::Node
-          <TopoViz::Node
+          @node={{node}}
-            @node={{node}}
+          @heightScale={{@heightScale}}
-            @heightScale={{@heightScale}}
+          @onAllocationSelect={{@onAllocationSelect}} />
-            @onAllocationSelect={{@onAllocationSelect}}
+      {{/each}}
-            @activeTaskGroup={{@activeTaskGroup}}
-            @activeJobId={{@activeJobId}} />
-        {{/each}}
-      {{/if}}
     </div>
   </div>
 </div>

ui/app/templates/components/topo-viz/node.hbs

@@ -1,10 +1,10 @@
 <div class="chart topo-viz-node {{unless this.allocations.length "is-empty"}}" {{did-insert this.reloadNode}}>
   <p>
-    <strong>{{@node.name}}</strong>
+    <strong>{{@node.node.name}}</strong>
     <span class="bumper-left">{{this.count}} Allocs</span>
-    <span class="bumper-left is-faded">{{@node.resources.memory}} MiB, {{@node.resources.cpu}} Mhz</span>
+    <span class="bumper-left is-faded">{{@node.memory}} MiB, {{@node.cpu}} Mhz</span>
   </p>
-  <svg class="chart" height="{{this.totalHeight}}px" {{did-insert this.render}} {{did-update this.updateRender @activeTaskGroup @activeJobId}} {{window-resize this.render}}>
+  <svg class="chart" height="{{this.totalHeight}}px" {{did-insert this.render}} {{did-update this.updateRender}} {{window-resize this.render}}>
     <defs>
       <clipPath id="{{this.maskId}}">
         <rect class="mask" x="0" y="0" width="{{this.dimensionsWidth}}px" height="{{this.maskHeight}}px" rx="2px" ry="2px" />
@@ -33,23 +33,23 @@
           {{/if}}
           {{#each this.data.memory key="allocation.id" as |memory|}}
             <g
-              class="bar {{memory.className}} {{if (eq this.activeAllocation memory.allocation) "is-active"}} {{if memory.isSelected "is-selected"}}"
+              class="bar {{memory.className}} {{if (eq this.activeAllocation memory.allocation) "is-active"}} {{if memory.allocation.isSelected "is-selected"}}"
               clip-path="url(#{{this.maskId}})"
-              data-allocation-id="{{memory.allocation.id}}"
+              data-allocation-id="{{memory.allocation.allocation.id}}"
               {{on "mouseenter" (fn this.highlightAllocation memory.allocation)}}
               {{on "click" (fn this.selectAllocation memory.allocation)}}>
               <rect
                 width="{{memory.width}}px"
-                height="{{if memory.isSelected this.selectedHeight this.height}}px"
+                height="{{if memory.allocation.isSelected this.selectedHeight this.height}}px"
                 x="{{memory.x}}px"
-                y="{{if memory.isSelected 0.5 0}}px"
+                y="{{if memory.allocation.isSelected 0.5 0}}px"
                 class="layer-0" />
               {{#if (or (eq memory.className "starting") (eq memory.className "pending"))}}
                 <rect
                   width="{{memory.width}}px"
-                  height="{{if memory.isSelected this.selectedHeight this.height}}px"
+                  height="{{if memory.allocation.isSelected this.selectedHeight this.height}}px"
                   x="{{memory.x}}px"
-                  y="{{if memory.isSelected 0.5 0}}px"
+                  y="{{if memory.allocation.isSelected 0.5 0}}px"
                   class="layer-1" />
               {{/if}}
             </g>
@@ -69,23 +69,23 @@
           {{/if}}
           {{#each this.data.cpu key="allocation.id" as |cpu|}}
             <g
-              class="bar {{cpu.className}} {{if (eq this.activeAllocation cpu.allocation) "is-active"}} {{if cpu.isSelected "is-selected"}}"
+              class="bar {{cpu.className}} {{if (eq this.activeAllocation cpu.allocation) "is-active"}} {{if cpu.allocation.isSelected "is-selected"}}"
               clip-path="url(#{{this.maskId}})"
-              data-allocation-id="{{cpu.allocation.id}}"
+              data-allocation-id="{{cpu.allocation.allocation.id}}"
               {{on "mouseenter" (fn this.highlightAllocation cpu.allocation)}}
               {{on "click" (fn this.selectAllocation cpu.allocation)}}>
               <rect
                 width="{{cpu.width}}px"
-                height="{{if cpu.isSelected this.selectedHeight this.height}}px"
+                height="{{if cpu.allocation.isSelected this.selectedHeight this.height}}px"
                 x="{{cpu.x}}px"
-                y="{{if cpu.isSelected this.selectedYOffset this.yOffset}}px"
+                y="{{if cpu.allocation.isSelected this.selectedYOffset this.yOffset}}px"
                 class="layer-0" />
               {{#if (or (eq cpu.className "starting") (eq cpu.className "pending"))}}
                 <rect
                   width="{{cpu.width}}px"
-                  height="{{if cpu.isSelected this.selectedHeight this.height}}px"
+                  height="{{if cpu.allocation.isSelected this.selectedHeight this.height}}px"
                   x="{{cpu.x}}px"
-                  y="{{if cpu.isSelected this.selectedYOffset this.yOffset}}px"
+                  y="{{if cpu.allocation.isSelected this.selectedYOffset this.yOffset}}px"
                   class="layer-1" />
               {{/if}}
             </g>