From def7b91d6c968b382d2a03608f922b4c20fdc721 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Tue, 25 Jun 2019 20:32:17 +0300 Subject: [PATCH 01/14] Migrate Word Cloud visualization to React --- client/app/lib/hooks/useQueryResult.js | 6 +- .../visualizations/word-cloud/Renderer.jsx | 121 +++++++++++++++ client/app/visualizations/word-cloud/index.js | 146 ++---------------- 3 files changed, 133 insertions(+), 140 deletions(-) create mode 100644 client/app/visualizations/word-cloud/Renderer.jsx diff --git a/client/app/lib/hooks/useQueryResult.js b/client/app/lib/hooks/useQueryResult.js index 6a7179b930..3541aa0387 100644 --- a/client/app/lib/hooks/useQueryResult.js +++ b/client/app/lib/hooks/useQueryResult.js @@ -2,9 +2,9 @@ import { useState, useEffect } from 'react'; function getQueryResultData(queryResult) { return { - columns: queryResult ? queryResult.getColumns() : [], - rows: queryResult ? queryResult.getData() : [], - filters: queryResult ? queryResult.getFilters() : [], + columns: queryResult ? queryResult.getColumns() || [] : [], + rows: queryResult ? queryResult.getData() || [] : [], + filters: queryResult ? queryResult.getFilters() || [] : [], }; } diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx new file mode 100644 index 0000000000..9dd51c4a63 --- /dev/null +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -0,0 +1,121 @@ +import d3 from 'd3'; +import cloud from 'd3-cloud'; +import { map, min, max, values } from 'lodash'; +import React from 'react'; +import { RendererPropTypes } from '@/visualizations'; + +function findWordFrequencies(data, columnName) { + const wordsHash = {}; + + data.forEach((row) => { + const wordsList = row[columnName].toString().split(' '); + wordsList.forEach((d) => { + if (d in wordsHash) { + wordsHash[d] += 1; + } else { + wordsHash[d] = 1; + } + }); + }); + + return wordsHash; +} + +// target domain: [t1, t2] +const MIN_WORD_SIZE = 10; +const MAX_WORD_SIZE = 100; + +function createScale(wordCounts) { + wordCounts = values(wordCounts); + + // source domain: [s1, s2] + const minCount = min(wordCounts); + const maxCount = max(wordCounts); + + // Edge case - if all words have the same count; just set middle size for all + if (minCount === maxCount) { + return () => (MAX_WORD_SIZE + MIN_WORD_SIZE) / 2; + } + + // v is value from source domain: + // s1 <= v <= s2. + // We need to fit it target domain: + // t1 <= v" <= t2 + // 1. offset source value to zero point: + // v' = v - s1 + // 2. offset source and target domains to zero point: + // s' = s2 - s1 + // t' = t2 - t1 + // 3. compute fraction: + // f = v' / s'; + // 0 <= f <= 1 + // 4. map f to target domain: + // v" = f * t' + t1; + // t1 <= v" <= t' + t1; + // t1 <= v" <= t2 (because t' = t2 - t1, so t2 = t' + t1) + + const sourceScale = maxCount - minCount; + const targetScale = MAX_WORD_SIZE - MIN_WORD_SIZE; + + return value => ((value - minCount) / sourceScale) * targetScale + MIN_WORD_SIZE; +} + +function render(container, data, options) { + let wordsHash = {}; + if (options.column) { + wordsHash = findWordFrequencies(data.rows, options.column); + } + + const scaleValue = createScale(wordsHash); + + const wordList = map(wordsHash, (count, key) => ({ + text: key, + size: scaleValue(count), + })); + + const fill = d3.scale.category20(); + const layout = cloud() + .size([500, 500]) + .words(wordList) + .padding(5) + .rotate(() => Math.floor(Math.random() * 2) * 90) + .font('Impact') + .fontSize(d => d.size); + + function draw(words) { + d3.select(container).selectAll('*').remove(); + + d3.select(container) + .append('svg') + .attr('width', layout.size()[0]) + .attr('height', layout.size()[1]) + .append('g') + .attr('transform', `translate(${layout.size()[0] / 2},${layout.size()[1] / 2})`) + .selectAll('text') + .data(words) + .enter() + .append('text') + .style('font-size', d => `${d.size}px`) + .style('font-family', 'Impact') + .style('fill', (d, i) => fill(i)) + .attr('text-anchor', 'middle') + .attr('transform', d => `translate(${[d.x, d.y]})rotate(${d.rotate})`) + .text(d => d.text); + } + + layout.on('end', draw); + + layout.start(); +} + +export default function Renderer({ data, options }) { + const containerRef = (container) => { + if (container) { + render(container, data, options); + } + }; + + return (
); +} + +Renderer.propTypes = RendererPropTypes; diff --git a/client/app/visualizations/word-cloud/index.js b/client/app/visualizations/word-cloud/index.js index 1f57689bcc..8edbba281b 100644 --- a/client/app/visualizations/word-cloud/index.js +++ b/client/app/visualizations/word-cloud/index.js @@ -1,145 +1,17 @@ -import d3 from 'd3'; -import cloud from 'd3-cloud'; -import { map, min, max, values } from 'lodash'; -import { angular2react } from 'angular2react'; import { registerVisualization } from '@/visualizations'; +import Renderer from './Renderer'; import Editor from './Editor'; -function findWordFrequencies(data, columnName) { - const wordsHash = {}; +export default function init() { + registerVisualization({ + type: 'WORD_CLOUD', + name: 'Word Cloud', + getOptions: options => ({ ...options }), + Renderer, + Editor, - data.forEach((row) => { - const wordsList = row[columnName].toString().split(' '); - wordsList.forEach((d) => { - if (d in wordsHash) { - wordsHash[d] += 1; - } else { - wordsHash[d] = 1; - } - }); - }); - - return wordsHash; -} - -// target domain: [t1, t2] -const MIN_WORD_SIZE = 10; -const MAX_WORD_SIZE = 100; - -function createScale(wordCounts) { - wordCounts = values(wordCounts); - - // source domain: [s1, s2] - const minCount = min(wordCounts); - const maxCount = max(wordCounts); - - // Edge case - if all words have the same count; just set middle size for all - if (minCount === maxCount) { - return () => (MAX_WORD_SIZE + MIN_WORD_SIZE) / 2; - } - - // v is value from source domain: - // s1 <= v <= s2. - // We need to fit it target domain: - // t1 <= v" <= t2 - // 1. offset source value to zero point: - // v' = v - s1 - // 2. offset source and target domains to zero point: - // s' = s2 - s1 - // t' = t2 - t1 - // 3. compute fraction: - // f = v' / s'; - // 0 <= f <= 1 - // 4. map f to target domain: - // v" = f * t' + t1; - // t1 <= v" <= t' + t1; - // t1 <= v" <= t2 (because t' = t2 - t1, so t2 = t' + t1) - - const sourceScale = maxCount - minCount; - const targetScale = MAX_WORD_SIZE - MIN_WORD_SIZE; - - return value => ((value - minCount) / sourceScale) * targetScale + MIN_WORD_SIZE; -} - -const WordCloudRenderer = { - restrict: 'E', - bindings: { - data: '<', - options: '<', - }, - controller($scope, $element) { - $element[0].style.display = 'block'; - - const update = () => { - const data = this.data.rows; - const options = this.options; - - let wordsHash = {}; - if (options.column) { - wordsHash = findWordFrequencies(data, options.column); - } - - const scaleValue = createScale(wordsHash); - - const wordList = map(wordsHash, (count, key) => ({ - text: key, - size: scaleValue(count), - })); - - const fill = d3.scale.category20(); - const layout = cloud() - .size([500, 500]) - .words(wordList) - .padding(5) - .rotate(() => Math.floor(Math.random() * 2) * 90) - .font('Impact') - .fontSize(d => d.size); - - function draw(words) { - d3.select($element[0]).selectAll('*').remove(); - - d3.select($element[0]) - .append('svg') - .attr('width', layout.size()[0]) - .attr('height', layout.size()[1]) - .append('g') - .attr('transform', `translate(${layout.size()[0] / 2},${layout.size()[1] / 2})`) - .selectAll('text') - .data(words) - .enter() - .append('text') - .style('font-size', d => `${d.size}px`) - .style('font-family', 'Impact') - .style('fill', (d, i) => fill(i)) - .attr('text-anchor', 'middle') - .attr('transform', d => `translate(${[d.x, d.y]})rotate(${d.rotate})`) - .text(d => d.text); - } - - layout.on('end', draw); - - layout.start(); - }; - - $scope.$watch('$ctrl.data', update); - $scope.$watch('$ctrl.options', update, true); - }, -}; - -export default function init(ngModule) { - ngModule.component('wordCloudRenderer', WordCloudRenderer); - - ngModule.run(($injector) => { - registerVisualization({ - type: 'WORD_CLOUD', - name: 'Word Cloud', - getOptions: options => ({ ...options }), - Renderer: angular2react('wordCloudRenderer', WordCloudRenderer, $injector), - Editor, - - defaultRows: 8, - }); + defaultRows: 8, }); } From e8eae8114edc7875c2a1e7736ac220fd2a874996 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Wed, 26 Jun 2019 21:39:04 +0300 Subject: [PATCH 02/14] Refine code; make Word cloud fit it's container --- .../visualizations/word-cloud/Renderer.jsx | 171 +++++++++--------- 1 file changed, 88 insertions(+), 83 deletions(-) diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index 9dd51c4a63..76abf94a23 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -1,14 +1,14 @@ import d3 from 'd3'; import cloud from 'd3-cloud'; -import { map, min, max, values } from 'lodash'; -import React from 'react'; +import { map, min, max, values, sortBy } from 'lodash'; +import React, { useMemo, useState, useEffect } from 'react'; import { RendererPropTypes } from '@/visualizations'; -function findWordFrequencies(data, columnName) { +function computeWordFrequencies(rows, column) { const wordsHash = {}; - data.forEach((row) => { - const wordsList = row[columnName].toString().split(' '); + rows.forEach((row) => { + const wordsList = row[column].toString().split(' '); wordsList.forEach((d) => { if (d in wordsHash) { wordsHash[d] += 1; @@ -21,101 +21,106 @@ function findWordFrequencies(data, columnName) { return wordsHash; } -// target domain: [t1, t2] -const MIN_WORD_SIZE = 10; -const MAX_WORD_SIZE = 100; +function prepareWords(rows, options) { + let result = []; -function createScale(wordCounts) { - wordCounts = values(wordCounts); + if (options.column) { + result = computeWordFrequencies(rows, options.column); + } - // source domain: [s1, s2] - const minCount = min(wordCounts); - const maxCount = max(wordCounts); + const counts = values(result); + const wordSize = d3.scale.linear() + .domain([min(counts), max(counts)]) + .range([10, 100]); // min/max word size - // Edge case - if all words have the same count; just set middle size for all - if (minCount === maxCount) { - return () => (MAX_WORD_SIZE + MIN_WORD_SIZE) / 2; - } + const color = d3.scale.category20(); - // v is value from source domain: - // s1 <= v <= s2. - // We need to fit it target domain: - // t1 <= v" <= t2 - // 1. offset source value to zero point: - // v' = v - s1 - // 2. offset source and target domains to zero point: - // s' = s2 - s1 - // t' = t2 - t1 - // 3. compute fraction: - // f = v' / s'; - // 0 <= f <= 1 - // 4. map f to target domain: - // v" = f * t' + t1; - // t1 <= v" <= t' + t1; - // t1 <= v" <= t2 (because t' = t2 - t1, so t2 = t' + t1) - - const sourceScale = maxCount - minCount; - const targetScale = MAX_WORD_SIZE - MIN_WORD_SIZE; - - return value => ((value - minCount) / sourceScale) * targetScale + MIN_WORD_SIZE; + result = map(result, (count, key) => ({ + text: key, + size: wordSize(count), + })); + + // add some attributes + result = map(result, (word, i) => ({ + ...word, + color: color(i), + angle: i % 2 * 90, // make it stable between renderings + })); + + return sortBy( + result, + [({ size }) => -size, ({ text }) => -text.length], // "size" desc, length("text") desc + ); } -function render(container, data, options) { - let wordsHash = {}; - if (options.column) { - wordsHash = findWordFrequencies(data.rows, options.column); - } +function scaleElement(node, container) { + node.style.transform = null; + const { width: nodeWidth, height: nodeHeight } = node.getBoundingClientRect(); + const { width: containerWidth, height: containerHeight } = container.getBoundingClientRect(); - const scaleValue = createScale(wordsHash); + const scaleX = containerWidth / nodeWidth; + const scaleY = containerHeight / nodeHeight; - const wordList = map(wordsHash, (count, key) => ({ - text: key, - size: scaleValue(count), - })); + node.style.transform = `scale(${Math.min(scaleX, scaleY)})`; + node.style.transformOrigin = 'left top'; +} - const fill = d3.scale.category20(); - const layout = cloud() - .size([500, 500]) - .words(wordList) - .padding(5) - .rotate(() => Math.floor(Math.random() * 2) * 90) +function createLayout() { + return cloud() + // make the area large enough to contain even very long words; word cloud will be placed in the center of the area + .size([10000, 10000]) + .padding(3) .font('Impact') - .fontSize(d => d.size); - - function draw(words) { - d3.select(container).selectAll('*').remove(); - - d3.select(container) - .append('svg') - .attr('width', layout.size()[0]) - .attr('height', layout.size()[1]) - .append('g') - .attr('transform', `translate(${layout.size()[0] / 2},${layout.size()[1] / 2})`) - .selectAll('text') - .data(words) - .enter() - .append('text') - .style('font-size', d => `${d.size}px`) - .style('font-family', 'Impact') - .style('fill', (d, i) => fill(i)) - .attr('text-anchor', 'middle') - .attr('transform', d => `translate(${[d.x, d.y]})rotate(${d.rotate})`) - .text(d => d.text); - } - - layout.on('end', draw); + .rotate(d => d.angle) + .fontSize(d => d.size) + .random(() => 0.5); // do not place words randomly - use compact layout +} - layout.start(); +function render(container, words) { + container = d3.select(container); + container.selectAll('*').remove(); + + const svg = container.append('svg'); + const g = svg.append('g'); + g.selectAll('text') + .data(words) + .enter() + .append('text') + .style('font-size', d => `${d.size}px`) + .style('font-family', d => d.font) + .style('fill', d => d.color) + .attr('text-anchor', 'middle') + .attr('transform', d => `translate(${[d.x, d.y]}) rotate(${d.rotate})`) + .text(d => d.text); + + // get real bounds of words and add some padding to ensure that everything is visible + const bounds = g.node().getBoundingClientRect(); + const width = bounds.width + 10; + const height = bounds.height + 10; + + svg.attr('width', width).attr('height', height); + g.attr('transform', `translate(${width / 2},${height / 2})`); + + scaleElement(svg.node(), container.node()); } export default function Renderer({ data, options }) { - const containerRef = (container) => { + const [container, setContainer] = useState(null); + const [words, setWords] = useState([]); + const layout = useMemo(createLayout, []); + + useEffect(() => { + layout.words(prepareWords(data.rows, options)).on('end', w => setWords(w)).start(); + return () => layout.on('end', null).stop(); + }, [layout, data, options, setWords]); + + useEffect(() => { if (container) { - render(container, data, options); + render(container, words); } - }; + }, [container, words]); - return (
); + return (
); } Renderer.propTypes = RendererPropTypes; From 9c2d9873229f775dfd8538502dab0d7bfebfca74 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Thu, 27 Jun 2019 21:31:17 +0300 Subject: [PATCH 03/14] getredash/redash#1414 Word Cloud: Ability to pass the word frequency --- .../app/visualizations/word-cloud/Editor.jsx | 51 ++++++++++++------- .../visualizations/word-cloud/Renderer.jsx | 40 ++++++++++----- client/app/visualizations/word-cloud/index.js | 7 ++- 3 files changed, 65 insertions(+), 33 deletions(-) diff --git a/client/app/visualizations/word-cloud/Editor.jsx b/client/app/visualizations/word-cloud/Editor.jsx index 3499ca603a..c600dca34b 100644 --- a/client/app/visualizations/word-cloud/Editor.jsx +++ b/client/app/visualizations/word-cloud/Editor.jsx @@ -1,30 +1,43 @@ -import { map } from 'lodash'; +import { map, merge } from 'lodash'; import React from 'react'; import Select from 'antd/lib/select'; import { EditorPropTypes } from '@/visualizations'; -const { Option } = Select; - export default function Editor({ options, data, onOptionsChange }) { - const onColumnChanged = (column) => { - const newOptions = { ...options, column }; - onOptionsChange(newOptions); + const optionsChanged = (newOptions) => { + onOptionsChange(merge({}, options, newOptions)); }; return ( -
- - -
+ +
+ + +
+
+ + +
+ ); } diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index 76abf94a23..bf142f85fc 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -1,31 +1,45 @@ import d3 from 'd3'; import cloud from 'd3-cloud'; -import { map, min, max, values, sortBy } from 'lodash'; +import { each, map, min, max, values, sortBy } from 'lodash'; import React, { useMemo, useState, useEffect } from 'react'; import { RendererPropTypes } from '@/visualizations'; function computeWordFrequencies(rows, column) { - const wordsHash = {}; - - rows.forEach((row) => { - const wordsList = row[column].toString().split(' '); - wordsList.forEach((d) => { - if (d in wordsHash) { - wordsHash[d] += 1; - } else { - wordsHash[d] = 1; - } + const result = {}; + + each(rows, (row) => { + const wordsList = row[column].toString().split(/\s/g); + each(wordsList, (d) => { + result[d] = (result[d] || 0) + 1; }); }); - return wordsHash; + return result; +} + +function getWordsWithFrequencies(rows, wordColumn, frequencyColumn) { + const result = {}; + + each(rows, (row) => { + const count = parseFloat(row[frequencyColumn]); + if (Number.isFinite(count) && (count > 0)) { + const word = row[wordColumn]; + result[word] = count; + } + }); + + return result; } function prepareWords(rows, options) { let result = []; if (options.column) { - result = computeWordFrequencies(rows, options.column); + if (options.frequenciesColumn) { + result = getWordsWithFrequencies(rows, options.column, options.frequenciesColumn); + } else { + result = computeWordFrequencies(rows, options.column); + } } const counts = values(result); diff --git a/client/app/visualizations/word-cloud/index.js b/client/app/visualizations/word-cloud/index.js index 8edbba281b..cdb7796902 100644 --- a/client/app/visualizations/word-cloud/index.js +++ b/client/app/visualizations/word-cloud/index.js @@ -3,11 +3,16 @@ import { registerVisualization } from '@/visualizations'; import Renderer from './Renderer'; import Editor from './Editor'; +const DEFAULT_OPTIONS = { + column: '', + frequenciesColumn: '', +}; + export default function init() { registerVisualization({ type: 'WORD_CLOUD', name: 'Word Cloud', - getOptions: options => ({ ...options }), + getOptions: options => ({ ...DEFAULT_OPTIONS, ...options }), Renderer, Editor, From 046dfc3d23b4eb147953d2ebd727567d72693d42 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Thu, 27 Jun 2019 21:42:06 +0300 Subject: [PATCH 04/14] Some fixes to renderer --- client/app/visualizations/word-cloud/Renderer.jsx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index bf142f85fc..eaf11d9ccd 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -81,8 +81,9 @@ function scaleElement(node, container) { function createLayout() { return cloud() - // make the area large enough to contain even very long words; word cloud will be placed in the center of the area - .size([10000, 10000]) + // make the area large enough to contain even very long words; word cloud will be placed in the center of the area + // TODO: dimensions probably should be larger, but `d3-cloud` has some performance issues related to these values + .size([5000, 5000]) .padding(3) .font('Impact') .rotate(d => d.angle) @@ -107,13 +108,11 @@ function render(container, words) { .attr('transform', d => `translate(${[d.x, d.y]}) rotate(${d.rotate})`) .text(d => d.text); - // get real bounds of words and add some padding to ensure that everything is visible - const bounds = g.node().getBoundingClientRect(); - const width = bounds.width + 10; - const height = bounds.height + 10; + const svgBounds = svg.node().getBoundingClientRect(); + const gBounds = g.node().getBoundingClientRect(); - svg.attr('width', width).attr('height', height); - g.attr('transform', `translate(${width / 2},${height / 2})`); + svg.attr('width', Math.ceil(gBounds.width)).attr('height', Math.ceil(gBounds.height)); + g.attr('transform', `translate(${svgBounds.left - gBounds.left},${svgBounds.top - gBounds.top})`); scaleElement(svg.node(), container.node()); } From f30607fb6f786cb6bfec41576477f83b2ae84053 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Thu, 27 Jun 2019 22:26:33 +0300 Subject: [PATCH 05/14] getredash/redash#1414 Limit on words length and count --- .../app/visualizations/word-cloud/Editor.jsx | 60 +++++++++++++++++-- .../visualizations/word-cloud/Renderer.jsx | 42 ++++++++++++- client/app/visualizations/word-cloud/index.js | 5 +- .../visualizations/word-cloud/renderer.less | 7 +++ 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 client/app/visualizations/word-cloud/renderer.less diff --git a/client/app/visualizations/word-cloud/Editor.jsx b/client/app/visualizations/word-cloud/Editor.jsx index c600dca34b..87663bae28 100644 --- a/client/app/visualizations/word-cloud/Editor.jsx +++ b/client/app/visualizations/word-cloud/Editor.jsx @@ -1,6 +1,8 @@ import { map, merge } from 'lodash'; import React from 'react'; import Select from 'antd/lib/select'; +import InputNumber from 'antd/lib/input-number'; +import * as Grid from 'antd/lib/grid'; import { EditorPropTypes } from '@/visualizations'; export default function Editor({ options, data, onOptionsChange }) { @@ -11,9 +13,9 @@ export default function Editor({ options, data, onOptionsChange }) { return (
- +
- +
+
+ + + + optionsChanged({ wordLengthLimit: { min: value > 0 ? value : null } })} + /> + + + optionsChanged({ wordLengthLimit: { max: value > 0 ? value : null } })} + /> + + +
+
+ + + + optionsChanged({ wordCountLimit: { min: value > 0 ? value : null } })} + /> + + + optionsChanged({ wordCountLimit: { max: value > 0 ? value : null } })} + /> + + +
); } diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index eaf11d9ccd..76e088a0c6 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -4,6 +4,8 @@ import { each, map, min, max, values, sortBy } from 'lodash'; import React, { useMemo, useState, useEffect } from 'react'; import { RendererPropTypes } from '@/visualizations'; +import './renderer.less'; + function computeWordFrequencies(rows, column) { const result = {}; @@ -31,6 +33,40 @@ function getWordsWithFrequencies(rows, wordColumn, frequencyColumn) { return result; } +function applyLimitsToWords(wordsHash, { wordLength, wordCount }) { + const result = {}; + + wordLength.min = Number.isFinite(wordLength.min) ? wordLength.min : null; + wordLength.max = Number.isFinite(wordLength.max) ? wordLength.max : null; + if (wordLength.min && wordLength.max && (wordLength.min > wordLength.max)) { + wordLength = { min: wordLength.max, max: wordLength.min }; // swap + } + + wordCount.min = Number.isFinite(wordCount.min) ? wordCount.min : null; + wordCount.max = Number.isFinite(wordCount.max) ? wordCount.max : null; + if (wordCount.min && wordCount.max && (wordCount.min > wordCount.max)) { + wordCount = { min: wordCount.max, max: wordCount.min }; // swap + } + + each(wordsHash, (count, word) => { + if (wordLength.min && (word.length < wordLength.min)) { + return; + } + if (wordLength.max && (word.length > wordLength.max)) { + return; + } + if (wordCount.min && (count < wordCount.min)) { + return; + } + if (wordCount.max && (count > wordCount.max)) { + return; + } + result[word] = count; + }); + + return result; +} + function prepareWords(rows, options) { let result = []; @@ -42,6 +78,11 @@ function prepareWords(rows, options) { } } + result = applyLimitsToWords(result, { + wordLength: options.wordLengthLimit, + wordCount: options.wordCountLimit, + }); + const counts = values(result); const wordSize = d3.scale.linear() .domain([min(counts), max(counts)]) @@ -76,7 +117,6 @@ function scaleElement(node, container) { const scaleY = containerHeight / nodeHeight; node.style.transform = `scale(${Math.min(scaleX, scaleY)})`; - node.style.transformOrigin = 'left top'; } function createLayout() { diff --git a/client/app/visualizations/word-cloud/index.js b/client/app/visualizations/word-cloud/index.js index cdb7796902..db5765adef 100644 --- a/client/app/visualizations/word-cloud/index.js +++ b/client/app/visualizations/word-cloud/index.js @@ -1,3 +1,4 @@ +import { merge } from 'lodash'; import { registerVisualization } from '@/visualizations'; import Renderer from './Renderer'; @@ -6,13 +7,15 @@ import Editor from './Editor'; const DEFAULT_OPTIONS = { column: '', frequenciesColumn: '', + wordLengthLimit: { min: null, max: null }, + wordCountLimit: { min: null, max: null }, }; export default function init() { registerVisualization({ type: 'WORD_CLOUD', name: 'Word Cloud', - getOptions: options => ({ ...DEFAULT_OPTIONS, ...options }), + getOptions: options => merge({}, DEFAULT_OPTIONS, options), Renderer, Editor, diff --git a/client/app/visualizations/word-cloud/renderer.less b/client/app/visualizations/word-cloud/renderer.less new file mode 100644 index 0000000000..f19b66df11 --- /dev/null +++ b/client/app/visualizations/word-cloud/renderer.less @@ -0,0 +1,7 @@ +.word-cloud-visualization-container { + overflow: hidden; + + svg { + transform-origin: left top; + } +} From 64b116d79fe7517cba11787a456e64d92e4c59c3 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Fri, 28 Jun 2019 21:15:28 +0300 Subject: [PATCH 06/14] Word Cloud: watch container resize and update visualization --- client/app/directives/resize-event.js | 45 +++---------------- client/app/pages/dashboards/dashboard.less | 1 + client/app/services/resizeObserver.js | 31 +++++++++++++ .../visualizations/word-cloud/Renderer.jsx | 8 ++++ .../visualizations/word-cloud/renderer.less | 7 ++- 5 files changed, 51 insertions(+), 41 deletions(-) create mode 100644 client/app/services/resizeObserver.js diff --git a/client/app/directives/resize-event.js b/client/app/directives/resize-event.js index a0c4d5bed0..396cf55b1a 100644 --- a/client/app/directives/resize-event.js +++ b/client/app/directives/resize-event.js @@ -1,48 +1,13 @@ -import { findIndex } from 'lodash'; - -const items = new Map(); - -function checkItems() { - items.forEach((item, node) => { - const bounds = node.getBoundingClientRect(); - // convert to int (because these numbers needed for comparisons), but preserve 1 decimal point - const width = Math.round(bounds.width * 10); - const height = Math.round(bounds.height * 10); - - if ( - (item.width !== width) || - (item.height !== height) - ) { - item.width = width; - item.height = height; - item.callback(node); - } - }); - - setTimeout(checkItems, 100); -} - -checkItems(); // ensure it was called only once! +import resizeObserver from '@/services/resizeObserver'; function resizeEvent() { return { restrict: 'A', link($scope, $element, attrs) { - const node = $element[0]; - if (!items.has(node)) { - items.set(node, { - callback: () => { - $scope.$evalAsync(attrs.resizeEvent); - }, - }); - - $scope.$on('$destroy', () => { - const index = findIndex(items, item => item.node === node); - if (index >= 0) { - items.splice(index, 1); // remove item - } - }); - } + const unwatch = resizeObserver($element[0], () => { + $scope.$evalAsync(attrs.resizeEvent); + }); + $scope.$on('$destroy', unwatch); }, }; } diff --git a/client/app/pages/dashboards/dashboard.less b/client/app/pages/dashboards/dashboard.less index 905bccfe4f..21f2cf2d60 100644 --- a/client/app/pages/dashboards/dashboard.less +++ b/client/app/pages/dashboards/dashboard.less @@ -83,6 +83,7 @@ .sunburst-visualization-container, .sankey-visualization-container, .map-visualization-container, + .word-cloud-visualization-container, .plotly-chart-container { position: absolute; left: 0; diff --git a/client/app/services/resizeObserver.js b/client/app/services/resizeObserver.js new file mode 100644 index 0000000000..810c6e40c5 --- /dev/null +++ b/client/app/services/resizeObserver.js @@ -0,0 +1,31 @@ +const items = new Map(); + +function checkItems() { + items.forEach((item, node) => { + const bounds = node.getBoundingClientRect(); + // convert to int (because these numbers needed for comparisons), but preserve 1 decimal point + const width = Math.round(bounds.width * 10); + const height = Math.round(bounds.height * 10); + + if ( + (item.width !== width) || + (item.height !== height) + ) { + item.width = width; + item.height = height; + item.callback(node); + } + }); + + setTimeout(checkItems, 100); +} + +checkItems(); // ensure it was called only once! + +export default function observe(node, callback) { + if (!items.has(node)) { + items.set(node, { callback }); + return () => items.delete(node); + } + return () => {}; +} diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index 76e088a0c6..741c863b73 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -2,6 +2,7 @@ import d3 from 'd3'; import cloud from 'd3-cloud'; import { each, map, min, max, values, sortBy } from 'lodash'; import React, { useMemo, useState, useEffect } from 'react'; +import resizeObserver from '@/services/resizeObserver'; import { RendererPropTypes } from '@/visualizations'; import './renderer.less'; @@ -173,6 +174,13 @@ export default function Renderer({ data, options }) { } }, [container, words]); + useEffect(() => resizeObserver(container, () => { + const svg = container.querySelector('svg'); + if (svg) { + scaleElement(svg, container); + } + }), [container]); + return (
); } diff --git a/client/app/visualizations/word-cloud/renderer.less b/client/app/visualizations/word-cloud/renderer.less index f19b66df11..aee6afcced 100644 --- a/client/app/visualizations/word-cloud/renderer.less +++ b/client/app/visualizations/word-cloud/renderer.less @@ -1,7 +1,12 @@ .word-cloud-visualization-container { overflow: hidden; + height: 400px; + display: flex; + align-items: center; + justify-content: center; svg { - transform-origin: left top; + transform-origin: center center; + flex: 0 0 auto; } } From 4b8ffe386bf850014b1bb54d54b79b74666775a1 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Sun, 30 Jun 2019 13:14:44 +0300 Subject: [PATCH 07/14] CR1 --- client/app/visualizations/word-cloud/Editor.jsx | 16 ++++++++-------- .../app/visualizations/word-cloud/Renderer.jsx | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/app/visualizations/word-cloud/Editor.jsx b/client/app/visualizations/word-cloud/Editor.jsx index 87663bae28..ea9e29da16 100644 --- a/client/app/visualizations/word-cloud/Editor.jsx +++ b/client/app/visualizations/word-cloud/Editor.jsx @@ -13,7 +13,7 @@ export default function Editor({ options, data, onOptionsChange }) { return (
- +
optionsChanged({ wordLengthLimit: { min: value > 0 ? value : null } })} @@ -56,7 +56,7 @@ export default function Editor({ options, data, onOptionsChange }) { optionsChanged({ wordLengthLimit: { max: value > 0 ? value : null } })} @@ -66,13 +66,13 @@ export default function Editor({ options, data, onOptionsChange }) {
optionsChanged({ wordCountLimit: { min: value > 0 ? value : null } })} @@ -81,7 +81,7 @@ export default function Editor({ options, data, onOptionsChange }) { optionsChanged({ wordCountLimit: { max: value > 0 ? value : null } })} diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index 741c863b73..f693df614e 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -1,6 +1,6 @@ import d3 from 'd3'; import cloud from 'd3-cloud'; -import { each, map, min, max, values, sortBy } from 'lodash'; +import { each, map, min, max, values, sortBy, toString } from 'lodash'; import React, { useMemo, useState, useEffect } from 'react'; import resizeObserver from '@/services/resizeObserver'; import { RendererPropTypes } from '@/visualizations'; @@ -11,7 +11,7 @@ function computeWordFrequencies(rows, column) { const result = {}; each(rows, (row) => { - const wordsList = row[column].toString().split(/\s/g); + const wordsList = toString(row[column]).split(/\s/g); each(wordsList, (d) => { result[d] = (result[d] || 0) + 1; }); From 28470f8c5d550167edfff98b51e4a0320027731d Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Sun, 30 Jun 2019 15:23:20 +0300 Subject: [PATCH 08/14] Word Cloud: preserve word color and rotation when applying filters --- .../visualizations/word-cloud/Renderer.jsx | 73 ++++++++----------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index f693df614e..4348e3d94b 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -1,6 +1,6 @@ import d3 from 'd3'; import cloud from 'd3-cloud'; -import { each, map, min, max, values, sortBy, toString } from 'lodash'; +import { each, filter, map, min, max, sortBy, toString } from 'lodash'; import React, { useMemo, useState, useEffect } from 'react'; import resizeObserver from '@/services/resizeObserver'; import { RendererPropTypes } from '@/visualizations'; @@ -26,7 +26,7 @@ function getWordsWithFrequencies(rows, wordColumn, frequencyColumn) { each(rows, (row) => { const count = parseFloat(row[frequencyColumn]); if (Number.isFinite(count) && (count > 0)) { - const word = row[wordColumn]; + const word = toString(row[wordColumn]); result[word] = count; } }); @@ -34,9 +34,7 @@ function getWordsWithFrequencies(rows, wordColumn, frequencyColumn) { return result; } -function applyLimitsToWords(wordsHash, { wordLength, wordCount }) { - const result = {}; - +function applyLimitsToWords(words, { wordLength, wordCount }) { wordLength.min = Number.isFinite(wordLength.min) ? wordLength.min : null; wordLength.max = Number.isFinite(wordLength.max) ? wordLength.max : null; if (wordLength.min && wordLength.max && (wordLength.min > wordLength.max)) { @@ -49,23 +47,17 @@ function applyLimitsToWords(wordsHash, { wordLength, wordCount }) { wordCount = { min: wordCount.max, max: wordCount.min }; // swap } - each(wordsHash, (count, word) => { - if (wordLength.min && (word.length < wordLength.min)) { - return; - } - if (wordLength.max && (word.length > wordLength.max)) { - return; - } - if (wordCount.min && (count < wordCount.min)) { - return; - } - if (wordCount.max && (count > wordCount.max)) { - return; - } - result[word] = count; + return filter(words, ({ text, count }) => { + const wordLengthFits = ( + (!wordLength.min || (text.length >= wordLength.min)) && + (!wordLength.max || (text.length <= wordLength.max)) + ); + const wordCountFits = ( + (!wordCount.min || (count >= wordCount.min)) && + (!wordCount.max || (count <= wordCount.max)) + ); + return wordLengthFits && wordCountFits; }); - - return result; } function prepareWords(rows, options) { @@ -77,36 +69,29 @@ function prepareWords(rows, options) { } else { result = computeWordFrequencies(rows, options.column); } + result = sortBy( + map(result, (count, text) => ({ text, count })), + [({ count }) => -count, ({ text }) => -text.length], // "count" desc, length("text") desc + ); } - result = applyLimitsToWords(result, { - wordLength: options.wordLengthLimit, - wordCount: options.wordCountLimit, - }); - - const counts = values(result); + // Add additional attributes + const counts = map(result, item => item.count); const wordSize = d3.scale.linear() .domain([min(counts), max(counts)]) .range([10, 100]); // min/max word size - const color = d3.scale.category20(); - result = map(result, (count, key) => ({ - text: key, - size: wordSize(count), - })); - - // add some attributes - result = map(result, (word, i) => ({ - ...word, - color: color(i), - angle: i % 2 * 90, // make it stable between renderings - })); - - return sortBy( - result, - [({ size }) => -size, ({ text }) => -text.length], // "size" desc, length("text") desc - ); + each(result, (item, index) => { + item.size = wordSize(item.count); + item.color = color(index); + item.angle = index % 2 * 90; // make it stable between renderings + }); + + return applyLimitsToWords(result, { + wordLength: options.wordLengthLimit, + wordCount: options.wordCountLimit, + }); } function scaleElement(node, container) { From e356aa7c85072c72f3dad5629104f8f8af2447e2 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 1 Jul 2019 12:01:15 +0300 Subject: [PATCH 09/14] CR1 --- client/app/lib/hooks/useQueryResult.js | 6 +++--- .../app/visualizations/word-cloud/Editor.jsx | 8 ++++---- .../visualizations/word-cloud/Renderer.jsx | 20 +++++++++---------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/client/app/lib/hooks/useQueryResult.js b/client/app/lib/hooks/useQueryResult.js index 3541aa0387..3918dd2fb9 100644 --- a/client/app/lib/hooks/useQueryResult.js +++ b/client/app/lib/hooks/useQueryResult.js @@ -2,9 +2,9 @@ import { useState, useEffect } from 'react'; function getQueryResultData(queryResult) { return { - columns: queryResult ? queryResult.getColumns() || [] : [], - rows: queryResult ? queryResult.getData() || [] : [], - filters: queryResult ? queryResult.getFilters() || [] : [], + columns: (queryResult && queryResult.getColumns()) || [], + rows: (queryResult && queryResult.getData()) || [], + filters: (queryResult && queryResult.getFilters()) || [], }; } diff --git a/client/app/visualizations/word-cloud/Editor.jsx b/client/app/visualizations/word-cloud/Editor.jsx index ea9e29da16..13ea524379 100644 --- a/client/app/visualizations/word-cloud/Editor.jsx +++ b/client/app/visualizations/word-cloud/Editor.jsx @@ -47,7 +47,7 @@ export default function Editor({ options, data, onOptionsChange }) { optionsChanged({ wordLengthLimit: { min: value > 0 ? value : null } })} @@ -56,7 +56,7 @@ export default function Editor({ options, data, onOptionsChange }) { optionsChanged({ wordLengthLimit: { max: value > 0 ? value : null } })} @@ -72,7 +72,7 @@ export default function Editor({ options, data, onOptionsChange }) { optionsChanged({ wordCountLimit: { min: value > 0 ? value : null } })} @@ -81,7 +81,7 @@ export default function Editor({ options, data, onOptionsChange }) { optionsChanged({ wordCountLimit: { max: value > 0 ? value : null } })} diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index 4348e3d94b..81e5debb81 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -37,15 +37,9 @@ function getWordsWithFrequencies(rows, wordColumn, frequencyColumn) { function applyLimitsToWords(words, { wordLength, wordCount }) { wordLength.min = Number.isFinite(wordLength.min) ? wordLength.min : null; wordLength.max = Number.isFinite(wordLength.max) ? wordLength.max : null; - if (wordLength.min && wordLength.max && (wordLength.min > wordLength.max)) { - wordLength = { min: wordLength.max, max: wordLength.min }; // swap - } wordCount.min = Number.isFinite(wordCount.min) ? wordCount.min : null; wordCount.max = Number.isFinite(wordCount.max) ? wordCount.max : null; - if (wordCount.min && wordCount.max && (wordCount.min > wordCount.max)) { - wordCount = { min: wordCount.max, max: wordCount.min }; // swap - } return filter(words, ({ text, count }) => { const wordLengthFits = ( @@ -159,12 +153,16 @@ export default function Renderer({ data, options }) { } }, [container, words]); - useEffect(() => resizeObserver(container, () => { - const svg = container.querySelector('svg'); - if (svg) { - scaleElement(svg, container); + useEffect(() => { + if (container) { + return resizeObserver(container, () => { + const svg = container.querySelector('svg'); + if (svg) { + scaleElement(svg, container); + } + }); } - }), [container]); + }, [container]); return (
); } From 5657a36adee0d106dde10ca3bf112d1aa5947246 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Mon, 1 Jul 2019 21:53:09 +0300 Subject: [PATCH 10/14] Add tests --- .../app/visualizations/word-cloud/Editor.jsx | 10 ++- .../visualizations/word_cloud_spec.js | 83 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 client/cypress/integration/visualizations/word_cloud_spec.js diff --git a/client/app/visualizations/word-cloud/Editor.jsx b/client/app/visualizations/word-cloud/Editor.jsx index 13ea524379..37cee6ab84 100644 --- a/client/app/visualizations/word-cloud/Editor.jsx +++ b/client/app/visualizations/word-cloud/Editor.jsx @@ -15,19 +15,21 @@ export default function Editor({ options, data, onOptionsChange }) {
@@ -46,6 +48,7 @@ export default function Editor({ options, data, onOptionsChange }) { { + beforeEach(() => { + cy.login(); + createQuery({ query: SQL }).then(({ id }) => { + cy.visit(`queries/${id}/source`); + cy.getByTestId('ExecuteButton').click(); + }); + }); + + it('creates visualization with automatic word frequencies', () => { + const visualizationName = 'Word Cloud (auto)'; + + cy.getByTestId('NewVisualization').click(); + cy.getByTestId('VisualizationType').click(); + cy.getByTestId('VisualizationType.WORD_CLOUD').click(); + cy.getByTestId('VisualizationName').clear().type(visualizationName); + + cy.getByTestId('WordCloud.WordsColumn').click(); + cy.getByTestId('WordCloud.WordsColumn.a').click(); + + cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 11); + + cy.getByTestId('EditVisualizationDialog').contains('button', 'Save').click(); + cy.getByTestId('QueryPageVisualizationTabs').contains('li', visualizationName).should('exist'); + }); + + it('creates visualization with word frequencies from another column', () => { + const visualizationName = 'Word Cloud (frequencies)'; + + cy.getByTestId('NewVisualization').click(); + cy.getByTestId('VisualizationType').click(); + cy.getByTestId('VisualizationType.WORD_CLOUD').click(); + cy.getByTestId('VisualizationName').clear().type(visualizationName); + + cy.getByTestId('WordCloud.WordsColumn').click(); + cy.getByTestId('WordCloud.WordsColumn.b').click(); + + cy.getByTestId('WordCloud.FrequenciesColumn').click(); + cy.getByTestId('WordCloud.FrequenciesColumn.c').click(); + + cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 5); + + cy.getByTestId('EditVisualizationDialog').contains('button', 'Save').click(); + cy.getByTestId('QueryPageVisualizationTabs').contains('li', visualizationName).should('exist'); + }); + + it('creates visualization with word length and frequencies limits', () => { + const visualizationName = 'Word Cloud (filters)'; + + cy.getByTestId('NewVisualization').click(); + cy.getByTestId('VisualizationType').click(); + cy.getByTestId('VisualizationType.WORD_CLOUD').click(); + cy.getByTestId('VisualizationName').clear().type(visualizationName); + + cy.getByTestId('WordCloud.WordsColumn').click(); + cy.getByTestId('WordCloud.WordsColumn.b').click(); + + cy.getByTestId('WordCloud.FrequenciesColumn').click(); + cy.getByTestId('WordCloud.FrequenciesColumn.c').click(); + + cy.getByTestId('WordCloud.WordLengthLimit.Min').clear().type('4'); + cy.getByTestId('WordCloud.WordLengthLimit.Max').clear().type('5'); + cy.getByTestId('WordCloud.WordCountLimit.Min').clear().type('1'); + cy.getByTestId('WordCloud.WordCountLimit.Max').clear().type('3'); + + cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 2); + + cy.getByTestId('EditVisualizationDialog').contains('button', 'Save').click(); + cy.getByTestId('QueryPageVisualizationTabs').contains('li', visualizationName).should('exist'); + }); +}); From a52ad52bbf97387532a55049b1490c513148cdd9 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Tue, 2 Jul 2019 12:58:02 +0300 Subject: [PATCH 11/14] Update tests --- .../edit_visualization_dialog_spec.js | 17 +++- .../visualizations/word_cloud_spec.js | 78 +++++++++---------- client/cypress/support/commands.js | 8 ++ 3 files changed, 61 insertions(+), 42 deletions(-) diff --git a/client/cypress/integration/visualizations/edit_visualization_dialog_spec.js b/client/cypress/integration/visualizations/edit_visualization_dialog_spec.js index aa8b61d486..4ce3bea175 100644 --- a/client/cypress/integration/visualizations/edit_visualization_dialog_spec.js +++ b/client/cypress/integration/visualizations/edit_visualization_dialog_spec.js @@ -22,8 +22,23 @@ describe('Edit visualization dialog', () => { it('opens Edit Visualization dialog', () => { cy.getByTestId('EditVisualization').click(); cy.getByTestId('EditVisualizationDialog').should('exist'); - // Default visualization should be selected + // Default `Table` visualization should be selected cy.getByTestId('VisualizationType').should('exist').should('contain', 'Table'); cy.getByTestId('VisualizationName').should('exist').should('have.value', 'Table'); }); + + it('creates visualization with custom name', () => { + const visualizationName = 'Custom name'; + + cy.clickThrough(` + NewVisualization + VisualizationType + VisualizationType.TABLE + `); + + cy.getByTestId('VisualizationName').clear().type(visualizationName); + + cy.getByTestId('EditVisualizationDialog').contains('button', 'Save').click(); + cy.getByTestId('QueryPageVisualizationTabs').contains('li', visualizationName).should('exist'); + }); }); diff --git a/client/cypress/integration/visualizations/word_cloud_spec.js b/client/cypress/integration/visualizations/word_cloud_spec.js index 3112e318ab..1df0c84b35 100644 --- a/client/cypress/integration/visualizations/word_cloud_spec.js +++ b/client/cypress/integration/visualizations/word_cloud_spec.js @@ -20,64 +20,60 @@ describe('Word Cloud', () => { }); it('creates visualization with automatic word frequencies', () => { - const visualizationName = 'Word Cloud (auto)'; + cy.clickThrough(` + NewVisualization + VisualizationType + VisualizationType.WORD_CLOUD - cy.getByTestId('NewVisualization').click(); - cy.getByTestId('VisualizationType').click(); - cy.getByTestId('VisualizationType.WORD_CLOUD').click(); - cy.getByTestId('VisualizationName').clear().type(visualizationName); - - cy.getByTestId('WordCloud.WordsColumn').click(); - cy.getByTestId('WordCloud.WordsColumn.a').click(); + WordCloud.WordsColumn + WordCloud.WordsColumn.a + `); cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 11); - cy.getByTestId('EditVisualizationDialog').contains('button', 'Save').click(); - cy.getByTestId('QueryPageVisualizationTabs').contains('li', visualizationName).should('exist'); + cy.percySnapshot('Visualizations - Word Cloud (Automatic word frequencies)'); }); it('creates visualization with word frequencies from another column', () => { - const visualizationName = 'Word Cloud (frequencies)'; - - cy.getByTestId('NewVisualization').click(); - cy.getByTestId('VisualizationType').click(); - cy.getByTestId('VisualizationType.WORD_CLOUD').click(); - cy.getByTestId('VisualizationName').clear().type(visualizationName); + cy.clickThrough(` + NewVisualization + VisualizationType + VisualizationType.WORD_CLOUD - cy.getByTestId('WordCloud.WordsColumn').click(); - cy.getByTestId('WordCloud.WordsColumn.b').click(); + WordCloud.WordsColumn + WordCloud.WordsColumn.b - cy.getByTestId('WordCloud.FrequenciesColumn').click(); - cy.getByTestId('WordCloud.FrequenciesColumn.c').click(); + WordCloud.FrequenciesColumn + WordCloud.FrequenciesColumn.c + `); cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 5); - cy.getByTestId('EditVisualizationDialog').contains('button', 'Save').click(); - cy.getByTestId('QueryPageVisualizationTabs').contains('li', visualizationName).should('exist'); + cy.percySnapshot('Visualizations - Word Cloud (Frequencies from another column)'); }); it('creates visualization with word length and frequencies limits', () => { - const visualizationName = 'Word Cloud (filters)'; - - cy.getByTestId('NewVisualization').click(); - cy.getByTestId('VisualizationType').click(); - cy.getByTestId('VisualizationType.WORD_CLOUD').click(); - cy.getByTestId('VisualizationName').clear().type(visualizationName); - - cy.getByTestId('WordCloud.WordsColumn').click(); - cy.getByTestId('WordCloud.WordsColumn.b').click(); - - cy.getByTestId('WordCloud.FrequenciesColumn').click(); - cy.getByTestId('WordCloud.FrequenciesColumn.c').click(); - - cy.getByTestId('WordCloud.WordLengthLimit.Min').clear().type('4'); - cy.getByTestId('WordCloud.WordLengthLimit.Max').clear().type('5'); - cy.getByTestId('WordCloud.WordCountLimit.Min').clear().type('1'); - cy.getByTestId('WordCloud.WordCountLimit.Max').clear().type('3'); + cy.clickThrough(` + NewVisualization + VisualizationType + VisualizationType.WORD_CLOUD + + WordCloud.WordsColumn + WordCloud.WordsColumn.b + + WordCloud.FrequenciesColumn + WordCloud.FrequenciesColumn.c + `); + + cy.fillInputs({ + 'WordCloud.WordLengthLimit.Min': '4', + 'WordCloud.WordLengthLimit.Max': '5', + 'WordCloud.WordCountLimit.Min': '1', + 'WordCloud.WordCountLimit.Max': '3', + }); cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 2); - cy.getByTestId('EditVisualizationDialog').contains('button', 'Save').click(); - cy.getByTestId('QueryPageVisualizationTabs').contains('li', visualizationName).should('exist'); + cy.percySnapshot('Visualizations - Word Cloud (With filters)'); }); }); diff --git a/client/cypress/support/commands.js b/client/cypress/support/commands.js index 83c2681ee5..57af84a8e3 100644 --- a/client/cypress/support/commands.js +++ b/client/cypress/support/commands.js @@ -1,5 +1,7 @@ import '@percy/cypress'; // eslint-disable-line import/no-extraneous-dependencies, import/no-unresolved +const { each } = Cypress._; + Cypress.Commands.add('login', (email = 'admin@redash.io', password = 'password') => cy.request({ url: '/login', method: 'POST', @@ -20,3 +22,9 @@ Cypress.Commands.add('clickThrough', (elements) => { .forEach(element => cy.getByTestId(element).click()); return undefined; }); + +Cypress.Commands.add('fillInputs', (elements) => { + each(elements, (value, testId) => { + cy.getByTestId(testId).clear().type(value); + }); +}); From 7c78ada6d9f37a2c6c96f3c057683d4f1906fb65 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Tue, 2 Jul 2019 13:23:43 +0300 Subject: [PATCH 12/14] Update tests --- client/app/services/resizeObserver.js | 59 ++++++++++++------- .../EditVisualizationDialog.jsx | 4 +- .../visualizations/word_cloud_spec.js | 11 +++- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/client/app/services/resizeObserver.js b/client/app/services/resizeObserver.js index 810c6e40c5..65c1c8f890 100644 --- a/client/app/services/resizeObserver.js +++ b/client/app/services/resizeObserver.js @@ -1,31 +1,50 @@ +/* global ResizeObserver */ + +function observeNative(node, callback) { + if ((typeof ResizeObserver === 'function') && node) { + const observer = new ResizeObserver(() => callback()); // eslint-disable-line compat/compat + observer.observe(node); + return () => observer.disconnect(); + } + return null; +} + const items = new Map(); function checkItems() { - items.forEach((item, node) => { - const bounds = node.getBoundingClientRect(); - // convert to int (because these numbers needed for comparisons), but preserve 1 decimal point - const width = Math.round(bounds.width * 10); - const height = Math.round(bounds.height * 10); + if (items.size > 0) { + items.forEach((item, node) => { + const bounds = node.getBoundingClientRect(); + // convert to int (because these numbers needed for comparisons), but preserve 1 decimal point + const width = Math.round(bounds.width * 10); + const height = Math.round(bounds.height * 10); - if ( - (item.width !== width) || - (item.height !== height) - ) { - item.width = width; - item.height = height; - item.callback(node); - } - }); + if ( + (item.width !== width) || + (item.height !== height) + ) { + item.width = width; + item.height = height; + item.callback(node); + } + }); - setTimeout(checkItems, 100); + setTimeout(checkItems, 100); + } } -checkItems(); // ensure it was called only once! - -export default function observe(node, callback) { - if (!items.has(node)) { +function observeFallback(node, callback) { + if (node && !items.has(node)) { + const shouldTrigger = items.size === 0; items.set(node, { callback }); + if (shouldTrigger) { + checkItems(); + } return () => items.delete(node); } - return () => {}; + return null; +} + +export default function observe(node, callback) { + return observeNative(node, callback) || observeFallback(node, callback) || (() => {}); } diff --git a/client/app/visualizations/EditVisualizationDialog.jsx b/client/app/visualizations/EditVisualizationDialog.jsx index 10c370a03e..db2a94384c 100644 --- a/client/app/visualizations/EditVisualizationDialog.jsx +++ b/client/app/visualizations/EditVisualizationDialog.jsx @@ -1,4 +1,4 @@ -import { extend, map, findIndex, isEqual } from 'lodash'; +import { extend, map, sortBy, findIndex, isEqual } from 'lodash'; import React, { useState, useMemo } from 'react'; import PropTypes from 'prop-types'; import Modal from 'antd/lib/modal'; @@ -158,7 +158,7 @@ function EditVisualizationDialog({ dialog, visualization, query, queryResult }) onChange={onTypeChanged} > {map( - registeredVisualizations, + sortBy(registeredVisualizations, ['type']), vis => {vis.name}, )} diff --git a/client/cypress/integration/visualizations/word_cloud_spec.js b/client/cypress/integration/visualizations/word_cloud_spec.js index 1df0c84b35..e49d30aa79 100644 --- a/client/cypress/integration/visualizations/word_cloud_spec.js +++ b/client/cypress/integration/visualizations/word_cloud_spec.js @@ -29,9 +29,12 @@ describe('Word Cloud', () => { WordCloud.WordsColumn.a `); + // Wait for proper initialization of visualization + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 11); - cy.percySnapshot('Visualizations - Word Cloud (Automatic word frequencies)'); + cy.percySnapshot('Visualizations - Word Cloud (Automatic word frequencies)', { widths: [1280] }); }); it('creates visualization with word frequencies from another column', () => { @@ -47,6 +50,9 @@ describe('Word Cloud', () => { WordCloud.FrequenciesColumn.c `); + // Wait for proper initialization of visualization + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 5); cy.percySnapshot('Visualizations - Word Cloud (Frequencies from another column)'); @@ -72,6 +78,9 @@ describe('Word Cloud', () => { 'WordCloud.WordCountLimit.Max': '3', }); + // Wait for proper initialization of visualization + cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 2); cy.percySnapshot('Visualizations - Word Cloud (With filters)'); From 17aacc1a02145103766c7c6c8a1ce188aebbfe96 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Tue, 2 Jul 2019 18:30:49 +0300 Subject: [PATCH 13/14] Update tests --- client/app/visualizations/word-cloud/Renderer.jsx | 2 +- client/app/visualizations/word-cloud/renderer.less | 9 +++++++++ .../integration/visualizations/word_cloud_spec.js | 8 ++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index 81e5debb81..56909c5f1c 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -105,7 +105,7 @@ function createLayout() { // TODO: dimensions probably should be larger, but `d3-cloud` has some performance issues related to these values .size([5000, 5000]) .padding(3) - .font('Impact') + .font('"Roboto Regular", sans-serif') .rotate(d => d.angle) .fontSize(d => d.size) .random(() => 0.5); // do not place words randomly - use compact layout diff --git a/client/app/visualizations/word-cloud/renderer.less b/client/app/visualizations/word-cloud/renderer.less index aee6afcced..74abb95e14 100644 --- a/client/app/visualizations/word-cloud/renderer.less +++ b/client/app/visualizations/word-cloud/renderer.less @@ -1,3 +1,12 @@ +@font-face { + font-family: "Roboto Regular"; + src: url("../../assets/fonts/roboto/Roboto-Regular-webfont.eot"); + src: url("../../assets/fonts/roboto/Roboto-Regular-webfont.eot?#iefix") format("embedded-opentype"), + url("../../assets/fonts/roboto/Roboto-Regular-webfont.woff") format("woff"), + url("../../assets/fonts/roboto/Roboto-Regular-webfont.ttf") format("truetype"), + url("../../assets/fonts/roboto/Roboto-Regular-webfont.svg") format("svg"); +} + .word-cloud-visualization-container { overflow: hidden; height: 400px; diff --git a/client/cypress/integration/visualizations/word_cloud_spec.js b/client/cypress/integration/visualizations/word_cloud_spec.js index e49d30aa79..27db65c157 100644 --- a/client/cypress/integration/visualizations/word_cloud_spec.js +++ b/client/cypress/integration/visualizations/word_cloud_spec.js @@ -30,11 +30,11 @@ describe('Word Cloud', () => { `); // Wait for proper initialization of visualization - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.wait(500); // eslint-disable-line cypress/no-unnecessary-waiting cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 11); - cy.percySnapshot('Visualizations - Word Cloud (Automatic word frequencies)', { widths: [1280] }); + cy.percySnapshot('Visualizations - Word Cloud (Automatic word frequencies)'); }); it('creates visualization with word frequencies from another column', () => { @@ -51,7 +51,7 @@ describe('Word Cloud', () => { `); // Wait for proper initialization of visualization - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.wait(500); // eslint-disable-line cypress/no-unnecessary-waiting cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 5); @@ -79,7 +79,7 @@ describe('Word Cloud', () => { }); // Wait for proper initialization of visualization - cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting + cy.wait(500); // eslint-disable-line cypress/no-unnecessary-waiting cy.getByTestId('VisualizationPreview').find('svg text').should('have.length', 2); From f50bf9a64351c74eca52c7b744b16d2dd5c6d3e2 Mon Sep 17 00:00:00 2001 From: Levko Kravets Date: Wed, 3 Jul 2019 12:56:59 +0300 Subject: [PATCH 14/14] Update tests (inject css) --- .../visualizations/word-cloud/Renderer.jsx | 4 +- .../visualizations/word-cloud/renderer.less | 9 ---- .../visualizations/word_cloud_spec.js | 45 ++++++++++++++++++- client/cypress/support/commands.js | 2 + webpack.config.js | 3 +- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/client/app/visualizations/word-cloud/Renderer.jsx b/client/app/visualizations/word-cloud/Renderer.jsx index 56909c5f1c..cbf6a50cdd 100644 --- a/client/app/visualizations/word-cloud/Renderer.jsx +++ b/client/app/visualizations/word-cloud/Renderer.jsx @@ -100,12 +100,14 @@ function scaleElement(node, container) { } function createLayout() { + const fontFamily = window.getComputedStyle(document.body).fontFamily; + return cloud() // make the area large enough to contain even very long words; word cloud will be placed in the center of the area // TODO: dimensions probably should be larger, but `d3-cloud` has some performance issues related to these values .size([5000, 5000]) .padding(3) - .font('"Roboto Regular", sans-serif') + .font(fontFamily) .rotate(d => d.angle) .fontSize(d => d.size) .random(() => 0.5); // do not place words randomly - use compact layout diff --git a/client/app/visualizations/word-cloud/renderer.less b/client/app/visualizations/word-cloud/renderer.less index 74abb95e14..aee6afcced 100644 --- a/client/app/visualizations/word-cloud/renderer.less +++ b/client/app/visualizations/word-cloud/renderer.less @@ -1,12 +1,3 @@ -@font-face { - font-family: "Roboto Regular"; - src: url("../../assets/fonts/roboto/Roboto-Regular-webfont.eot"); - src: url("../../assets/fonts/roboto/Roboto-Regular-webfont.eot?#iefix") format("embedded-opentype"), - url("../../assets/fonts/roboto/Roboto-Regular-webfont.woff") format("woff"), - url("../../assets/fonts/roboto/Roboto-Regular-webfont.ttf") format("truetype"), - url("../../assets/fonts/roboto/Roboto-Regular-webfont.svg") format("svg"); -} - .word-cloud-visualization-container { overflow: hidden; height: 400px; diff --git a/client/cypress/integration/visualizations/word_cloud_spec.js b/client/cypress/integration/visualizations/word_cloud_spec.js index 27db65c157..ff687b31e2 100644 --- a/client/cypress/integration/visualizations/word_cloud_spec.js +++ b/client/cypress/integration/visualizations/word_cloud_spec.js @@ -1,7 +1,9 @@ -/* global cy */ +/* global cy, Cypress */ import { createQuery } from '../../support/redash-api'; +const { map } = Cypress._; + const SQL = ` SELECT 'Lorem ipsum dolor' AS a, 'ipsum' AS b, 2 AS c UNION ALL SELECT 'Lorem sit amet' AS a, 'amet' AS b, 2 AS c UNION ALL @@ -10,6 +12,46 @@ const SQL = ` SELECT 'sed eiusmod tempor' AS a, 'tempor' AS b, 7 AS c `; +// Hack to fix Cypress -> Percy communication +// Word Cloud uses `font-family` defined in CSS with a lot of fallbacks, so +// it's almost impossible to know which font will be used on particular machine/browser. +// In Cypress browser it could be one font, in Percy - another. +// The issue is in how Percy takes screenshots: it takes a DOM/CSS/assets snapshot in Cypress, +// copies it to own servers and restores in own browsers. Word Cloud computes its layout +// using Cypress font, sets absolute positions for elements (in pixels), and when it is restored +// on Percy machines (with another font) - visualization gets messed up. +// Solution: explicitly provide some font that will be 100% the same on all CI machines. In this +// case, it's "Roboto" just because it's in the list of fallback fonts and we already have this +// webfont in assets folder (so browser can load it). +function injectFont(document) { + const style = document.createElement('style'); + style.setAttribute('id', 'percy-fix'); + style.setAttribute('type', 'text/css'); + + const fonts = [ + ['Roboto', 'Roboto-Light-webfont', 300], + ['Roboto', 'Roboto-Regular-webfont', 400], + ['Roboto', 'Roboto-Medium-webfont', 500], + ['Roboto', 'Roboto-Bold-webfont', 700], + ]; + + const basePath = '/static/fonts/roboto/'; + + // `insertRule` does not load font for some reason. Using text content works ¯\_(ツ)_/¯ + style.appendChild(document.createTextNode(map(fonts, ([fontFamily, fileName, fontWeight]) => (` + @font-face { + font-family: "${fontFamily}"; + font-weight: ${fontWeight}; + src: url("${basePath}${fileName}.eot"); + src: url("${basePath}${fileName}.eot?#iefix") format("embedded-opentype"), + url("${basePath}${fileName}.woff") format("woff"), + url("${basePath}${fileName}.ttf") format("truetype"), + url("${basePath}${fileName}.svg") format("svg"); + } + `)).join('\n\n'))); + document.getElementsByTagName('head')[0].appendChild(style); +} + describe('Word Cloud', () => { beforeEach(() => { cy.login(); @@ -17,6 +59,7 @@ describe('Word Cloud', () => { cy.visit(`queries/${id}/source`); cy.getByTestId('ExecuteButton').click(); }); + cy.document().then(injectFont); }); it('creates visualization with automatic word frequencies', () => { diff --git a/client/cypress/support/commands.js b/client/cypress/support/commands.js index 57af84a8e3..97efba27f2 100644 --- a/client/cypress/support/commands.js +++ b/client/cypress/support/commands.js @@ -1,3 +1,5 @@ +/* global Cypress */ + import '@percy/cypress'; // eslint-disable-line import/no-extraneous-dependencies, import/no-unresolved const { each } = Cypress._; diff --git a/webpack.config.js b/webpack.config.js index f0bcf90221..a413079ea7 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -75,7 +75,8 @@ const config = { { from: "client/app/unsupported.html" }, { from: "client/app/unsupportedRedirect.js" }, { from: "client/app/assets/css/*.css", to: "styles/", flatten: true }, - { from: "node_modules/jquery/dist/jquery.min.js", to: "js/jquery.min.js" } + { from: "node_modules/jquery/dist/jquery.min.js", to: "js/jquery.min.js" }, + { from: "client/app/assets/fonts", to: "fonts/" }, ]) ], optimization: {