From 5737907b691724ce3b4e2f59af76eabb2a0f6d44 Mon Sep 17 00:00:00 2001 From: Marcos Martinez Date: Sun, 6 Dec 2020 00:58:19 -0800 Subject: [PATCH] Initial workflow with .csv download --- .../src/components/AppContent.js | 13 +++- .../src/components/content/Step1.js | 18 ++--- .../src/components/content/Step2.js | 11 +-- .../src/components/content/Step3.js | 17 ++-- .../src/components/content/TopicsSelection.js | 49 ------------ .../src/components/content/TopicsSelector.js | 61 +++++++++++++++ phs-gdc-dashboard/src/constants.js | 11 ++- phs-gdc-dashboard/src/resources/data1.json | 63 +++++++++++++++ phs-gdc-dashboard/src/resources/data2.json | 34 +++++++- .../src/services/dataCommonsService.js | 16 +--- .../src/utils/dataCommonsUtils.js | 78 ++++++++++++++++--- 11 files changed, 260 insertions(+), 111 deletions(-) delete mode 100644 phs-gdc-dashboard/src/components/content/TopicsSelection.js create mode 100644 phs-gdc-dashboard/src/components/content/TopicsSelector.js create mode 100644 phs-gdc-dashboard/src/resources/data1.json diff --git a/phs-gdc-dashboard/src/components/AppContent.js b/phs-gdc-dashboard/src/components/AppContent.js index 1e3dcc0..9c7b6e1 100644 --- a/phs-gdc-dashboard/src/components/AppContent.js +++ b/phs-gdc-dashboard/src/components/AppContent.js @@ -5,6 +5,7 @@ import Paper from "@material-ui/core/Paper"; import Step1 from "./content/Step1"; import Step2 from "./content/Step2"; import Step3 from "./content/Step3"; +import {INDEX_VARIABLE_NAME_DEFAULT} from "../constants"; const useStyles = makeStyles((theme) => ({ root: { @@ -24,23 +25,29 @@ export default function AppContent() { const classes = useStyles(); const [phsVariableValues, setPhsVariableValues] = React.useState(''); + const [phsVariableName, setPhsVariableName] = React.useState(INDEX_VARIABLE_NAME_DEFAULT); + const [dcVariableNames, setDcVariableNames] = React.useState(''); return (
- + - + - + diff --git a/phs-gdc-dashboard/src/components/content/Step1.js b/phs-gdc-dashboard/src/components/content/Step1.js index 0ab827e..3a560ad 100644 --- a/phs-gdc-dashboard/src/components/content/Step1.js +++ b/phs-gdc-dashboard/src/components/content/Step1.js @@ -6,13 +6,9 @@ import makeStyles from "@material-ui/core/styles/makeStyles"; import MenuItem from "@material-ui/core/MenuItem"; import FormHelperText from "@material-ui/core/FormHelperText"; import TextField from "@material-ui/core/TextField"; -import Autocomplete from "@material-ui/lab/Autocomplete/Autocomplete"; -import myData from "../../resources/data2"; -import Checkbox from "@material-ui/core/Checkbox/Checkbox"; -import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; -import ChevronRightIcon from '@material-ui/icons/ChevronRight'; import CheckBoxOutlineBlankIcon from "@material-ui/core/SvgIcon/SvgIcon"; import CheckBoxIcon from '@material-ui/icons/CheckBox'; +import {INDEX_VARIABLE_NAME_DEFAULT, INDEX_VARIABLES} from "../../constants"; export default function Step1(props) { @@ -32,15 +28,15 @@ export default function Step1(props) { })); const classes = useStyles(); - const [variable, setVariable] = React.useState(''); + const [variable, setVariable] = React.useState(INDEX_VARIABLE_NAME_DEFAULT); const [values, setValues] = React.useState(''); const handleChangeVariableSelect = (event) => { setVariable(event.target.value); + props.setPhsVariableName(event.target.value); }; const handleChangeValuesField = (event) => { - console.log('on change invoked') setValues(event.target.value); if (values != null && values.trim().length > 0) { let valuesArray = values.split(/\r?\n/); @@ -51,7 +47,7 @@ export default function Step1(props) { } }; - const phsVariables = ["state", "county", "city", "zip code"]; + const phsIndexVariables = INDEX_VARIABLES; return (
@@ -67,7 +63,7 @@ export default function Step1(props) { variant="outlined" className={classes.selectVariable} label="Variable"> - {phsVariables.map((phsVar, index) => + {phsIndexVariables.map((phsVar, index) => {phsVar} )}; @@ -77,8 +73,8 @@ export default function Step1(props) { id="standard-multiline-flexible" label="Variable values" multiline - rowsMax={10} - rows={6} + rowsMax={20} + rows={10} value={values} variant="outlined" onBlur={handleChangeValuesField} diff --git a/phs-gdc-dashboard/src/components/content/Step2.js b/phs-gdc-dashboard/src/components/content/Step2.js index 0e6a91a..cf3b18b 100644 --- a/phs-gdc-dashboard/src/components/content/Step2.js +++ b/phs-gdc-dashboard/src/components/content/Step2.js @@ -1,8 +1,8 @@ import React from "react"; import makeStyles from "@material-ui/core/styles/makeStyles"; -import TopicsSelection from "./TopicsSelection"; +import TopicsSelector from "./TopicsSelector"; -export default function Step1(props) { +export default function Step2(props) { const useStyles = makeStyles((theme) => ({ topics: { @@ -13,11 +13,6 @@ export default function Step1(props) { })); const classes = useStyles(); - const [topic, setTopic] = React.useState(''); - - const handleChange = (event) => { - setTopic(event.target.value); - }; return (
@@ -25,7 +20,7 @@ export default function Step1(props) {

Select the Data Commons variables you want to retrieve values from

{/**/}
- +
); diff --git a/phs-gdc-dashboard/src/components/content/Step3.js b/phs-gdc-dashboard/src/components/content/Step3.js index 9be51f7..3630c0a 100644 --- a/phs-gdc-dashboard/src/components/content/Step3.js +++ b/phs-gdc-dashboard/src/components/content/Step3.js @@ -5,10 +5,10 @@ import DialogTitle from "@material-ui/core/DialogTitle"; import DialogContent from "@material-ui/core/DialogContent"; import DialogContentText from "@material-ui/core/DialogContentText"; import {getPlaceStatistics} from "../../services/dataCommonsService"; -import {toTabularJsonData} from "../../utils/dataCommonsUtils"; +import {indexVariableValueToDcid, toTabularJsonData} from "../../utils/dataCommonsUtils"; import {jsonToCsv} from "../../utils/utils"; -export default function Step1(props) { +export default function Step3(props) { const [open, setOpen] = React.useState(false); @@ -35,11 +35,11 @@ export default function Step1(props) { }; function getCsvData() { - console.log(props.phsVariableValues); - let uniquePhsVariableValues = [...new Set(props.phsVariableValues)]; - console.log(uniquePhsVariableValues) - return getPlaceStatistics(uniquePhsVariableValues, ["Count_Person", "Median_Income_Person"]).then((data) => { - let tabJsonData = toTabularJsonData(data, "zip code", props.phsVariableValues); + console.log('dcVariableNames', props.dcVariableNames); + let phsVariableDcids = props.phsVariableValues.map(v => indexVariableValueToDcid(v, props.phsVariableName)) + let uniquePhsVariableDcids = [...new Set(phsVariableDcids)]; + return getPlaceStatistics(uniquePhsVariableDcids, props.dcVariableNames).then((data) => { + let tabJsonData = toTabularJsonData(data, props.phsVariableName, props.phsVariableValues); return jsonToCsv(tabJsonData); }) .catch((error) => { @@ -67,8 +67,7 @@ export default function Step1(props) { return (

{props.title}

- {props.phsVariableValues} -

Download the Data Commons data and use it from your R project

+

Download the Data Commons data and use them in your project

diff --git a/phs-gdc-dashboard/src/components/content/TopicsSelection.js b/phs-gdc-dashboard/src/components/content/TopicsSelection.js deleted file mode 100644 index 7bf7653..0000000 --- a/phs-gdc-dashboard/src/components/content/TopicsSelection.js +++ /dev/null @@ -1,49 +0,0 @@ -/* eslint-disable no-use-before-define */ - -import React from 'react'; -import Checkbox from '@material-ui/core/Checkbox'; -import TextField from '@material-ui/core/TextField'; -import Autocomplete from '@material-ui/lab/Autocomplete'; -import CheckBoxOutlineBlankIcon from '@material-ui/icons/CheckBoxOutlineBlank'; -import CheckBoxIcon from '@material-ui/icons/CheckBox'; -import myData from './../../resources/data2.json'; -import makeStyles from "@material-ui/core/styles/makeStyles"; - -const icon = ; -const checkedIcon = ; - -export default function TopicsSelection() { - - const useStyles = makeStyles((theme) => ({ - autoComplete: { - maxWidth: '100%', - }, - })); - - const classes = useStyles(); - - return ( - option.category} - getOptionLabel={(option) => option.name} - renderOption={(option, { selected }) => ( - - - {option.name} - - )} - renderInput={(params) => ( - - )} - /> - ); -} diff --git a/phs-gdc-dashboard/src/components/content/TopicsSelector.js b/phs-gdc-dashboard/src/components/content/TopicsSelector.js new file mode 100644 index 0000000..ba3c1e6 --- /dev/null +++ b/phs-gdc-dashboard/src/components/content/TopicsSelector.js @@ -0,0 +1,61 @@ +/* eslint-disable no-use-before-define */ + +import React from 'react'; +import Checkbox from '@material-ui/core/Checkbox'; +import TextField from '@material-ui/core/TextField'; +import Autocomplete from '@material-ui/lab/Autocomplete'; +import CheckBoxOutlineBlankIcon from '@material-ui/icons/CheckBoxOutlineBlank'; +import CheckBoxIcon from '@material-ui/icons/CheckBox'; +import myData from './../../resources/data2.json'; +import makeStyles from "@material-ui/core/styles/makeStyles"; + +const icon = ; +const checkedIcon = ; + +export default function TopicsSelector(props) { + + const useStyles = makeStyles((theme) => ({ + autoComplete: { + maxWidth: '100%', + }, + })); + + const classes = useStyles(); + + //const [selectedVariableNames, setSelectedVariableNames] = React.useState([]); + + const handleChange = (values) => { + //setSelectedVariableNames(values.map(value => value.name)); + props.setDcVariableNames(values.map(value => value.name)); + }; + + return ( + option.category} + + onChange={(event, values) => handleChange(values)} + //value={selectedVariableNames} + + getOptionLabel={(option) => option.label} + //getOptionSelected={(option) => option.name} + renderOption={(option, {selected}) => ( + + + {option.label} + + )} + renderInput={(params) => ( + + )} + /> + ); +} diff --git a/phs-gdc-dashboard/src/constants.js b/phs-gdc-dashboard/src/constants.js index 763418f..beca4a1 100644 --- a/phs-gdc-dashboard/src/constants.js +++ b/phs-gdc-dashboard/src/constants.js @@ -1 +1,10 @@ -export const DC_API_BASE_URL = "https://api.datacommons.org/"; \ No newline at end of file +export const DC_API_BASE_URL = "https://api.datacommons.org/"; + +export const INDEX_VARIABLE_NAME_STATE = "state"; +export const INDEX_VARIABLE_NAME_COUNTY = "county"; +export const INDEX_VARIABLE_NAME_CITY = "city"; +export const INDEX_VARIABLE_NAME_ZIP_CODE = "zip code"; +export const INDEX_VARIABLES = [INDEX_VARIABLE_NAME_STATE, INDEX_VARIABLE_NAME_COUNTY, INDEX_VARIABLE_NAME_CITY, INDEX_VARIABLE_NAME_ZIP_CODE]; +export const INDEX_VARIABLE_NAME_DEFAULT = INDEX_VARIABLE_NAME_ZIP_CODE; + +export const NOT_AVAILABLE_VALUE = "NA"; \ No newline at end of file diff --git a/phs-gdc-dashboard/src/resources/data1.json b/phs-gdc-dashboard/src/resources/data1.json new file mode 100644 index 0000000..f6a85ae --- /dev/null +++ b/phs-gdc-dashboard/src/resources/data1.json @@ -0,0 +1,63 @@ +[ + { + "name": "Count_Person", + "label": "Count_Person", + "category": "Demographics" + }, + { + "name": "Median_Income_Person", + "label": "Median_Income_Person", + "category": "Demographics" + }, + { + "name": "Median_Age_Person", + "label": "Median_Age_Person", + "category": "Demographics" + }, + { + "name": "UnemploymentRate_Person", + "label": "UnemploymentRate_Person", + "category": "Demographics" + }, + { + "name": "Count_Household", + "label": "Count_Household", + "category": "Demographics" + }, + { + "name": "LifeExpectancy_Person", + "label": "LifeExpectancy_Person", + "category": "Demographics" + }, + { + "name": "Count_CriminalActivities_CombinedCrime", + "label": "Count_CriminalActivities_CombinedCrime", + "category": "Crime" + }, + { + "name": "Count_Death", + "label": "Count_Death", + "category": "Health" + }, + { + "name": "Count_MedicareEnrollee", + "label": "Count_MedicareEnrollee", + "category": "Health" + }, + { + "name": "Count_Worker", + "label": "Count_Worker", + "category": "Employment" + }, + { + "name": "Amount_Emissions_CarbonDioxide_PerCapita", + "label": "Amount_Emissions_CarbonDioxide_PerCapita", + "category": "Environment" + }, + { + "name": "Median_Income_Household", + "label": "Median_Income_Household", + "category": "Household" + } + +] diff --git a/phs-gdc-dashboard/src/resources/data2.json b/phs-gdc-dashboard/src/resources/data2.json index 1d2c8c4..bd00744 100644 --- a/phs-gdc-dashboard/src/resources/data2.json +++ b/phs-gdc-dashboard/src/resources/data2.json @@ -1,12 +1,38 @@ [ { - "name": "Person Count", - "variable": "Count_Person", + "name": "Count_Person", + "label": "Count_Person", "category": "Demographics" }, { - "name": "Median Income (per person)", - "variable": "Median_Income_Person", + "name": "Median_Income_Person", + "label": "Median_Income_Person", "category": "Demographics" + }, + { + "name": "Median_Age_Person", + "label": "Median_Age_Person", + "category": "Demographics" + }, + { + "name": "Count_Household", + "label": "Count_Household", + "category": "Demographics" + }, + { + "name": "Count_CriminalActivities_CombinedCrime", + "label": "Count_CriminalActivities_CombinedCrime", + "category": "Crime" + }, + { + "name": "Count_MedicareEnrollee", + "label": "Count_MedicareEnrollee", + "category": "Health" + }, + { + "name": "Median_Income_Household", + "label": "Median_Income_Household", + "category": "Household" } + ] diff --git a/phs-gdc-dashboard/src/services/dataCommonsService.js b/phs-gdc-dashboard/src/services/dataCommonsService.js index 7ed72f2..4c836c7 100644 --- a/phs-gdc-dashboard/src/services/dataCommonsService.js +++ b/phs-gdc-dashboard/src/services/dataCommonsService.js @@ -10,24 +10,12 @@ import {DC_API_BASE_URL} from "../constants"; * @param statVars * @constructor */ -// export function GetPlaceStatistics(places, statVars) { -// let url = DC_API_BASE_URL + "stat/all"; -// console.log(url); -// const requestOptions = { -// method: 'POST', -// headers: {'Content-Type': 'application/json'}, -// body: JSON.stringify({places: places, statVars: statVars}) -// }; -// return fetch(url, requestOptions); -// }; - -export function getPlaceStatistics(places, statVars) { +export function getPlaceStatistics(placeDcids, statVars) { let url = DC_API_BASE_URL + "stat/all"; - console.log(url); const requestOptions = { method: 'POST', headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({places: places, statVars: statVars}) + body: JSON.stringify({places: placeDcids, statVars: statVars}) }; return fetch(url, requestOptions).then(response => { // Check if the request is 200 diff --git a/phs-gdc-dashboard/src/utils/dataCommonsUtils.js b/phs-gdc-dashboard/src/utils/dataCommonsUtils.js index 72a05ab..c645443 100644 --- a/phs-gdc-dashboard/src/utils/dataCommonsUtils.js +++ b/phs-gdc-dashboard/src/utils/dataCommonsUtils.js @@ -3,14 +3,24 @@ */ +import { + INDEX_VARIABLE_NAME_CITY, + INDEX_VARIABLE_NAME_COUNTY, + INDEX_VARIABLE_NAME_STATE, + INDEX_VARIABLE_NAME_ZIP_CODE, NOT_AVAILABLE_VALUE +} from "../constants"; + /** * Transforms data returned from the Data Commons /stat/all API to Json in tabular format * See https://docs.datacommons.org/api/rest/stat_all.html for some examples of input json data. * * @param jsonData - * @param placeVarName Name of the place variable. It will be used as the column name for the place data in the output + * @param phsVariableName + * @param phsVariableValues */ -export function toTabularJsonData(jsonData, placeVarName, phsVariableValues) { +export function toTabularJsonData(jsonData, phsVariableName, phsVariableValues) { + + console.log(jsonData); let tabularJsonData = []; let statVars = {}; @@ -19,30 +29,41 @@ export function toTabularJsonData(jsonData, placeVarName, phsVariableValues) { // To improve performance, we assume that a particular variable has the same years for all location identifiers let firstKey = (Object.keys(jsonData['placeData']))[0]; for (let varName in jsonData['placeData'][firstKey]['statVarData']) { - let year = getLatestYear(jsonData['placeData'][firstKey]['statVarData'][varName]["sourceSeries"][0]["val"]) - statVars[varName] = year; + if (jsonData['placeData'][firstKey]['statVarData'][varName]["sourceSeries"]) { + let year = getLatestYear(jsonData['placeData'][firstKey]['statVarData'][varName]["sourceSeries"][0]["val"]) + statVars[varName] = year; + } + else { // no data + statVars[varName] = null; + } } + console.log(statVars); let rows = {} // Generate rows and index them by phsVariableValue (placeId) for (let placeId in jsonData['placeData']) { - let placeValue = placeId; // TODO: extract value - let row = {[placeVarName] : placeValue}; + let placeValue = dcidToIndexVariableValue(placeId, phsVariableName); + let row = {[phsVariableName]: placeValue}; for (let statVarName in statVars) { let year = statVars[statVarName]; - row[statVarName] = jsonData['placeData'][placeId]['statVarData'][statVarName]["sourceSeries"][0]["val"][year]; + if (jsonData['placeData'][placeId]['statVarData'][statVarName]["sourceSeries"]) { + row[statVarName] = jsonData['placeData'][placeId]['statVarData'][statVarName]["sourceSeries"][0]["val"][year]; + } + else { + row[statVarName] = NOT_AVAILABLE_VALUE; + } } - rows[placeId] = row; + rows[placeValue] = row; } // Generate the final tabular data (it may contain duplicated rows) - phsVariableValues.forEach(placeId => { - tabularJsonData.push(rows[placeId]); + phsVariableValues.forEach(placeValue => { + tabularJsonData.push(rows[placeValue]); }); return tabularJsonData; -} +}; /** * Given an object with key-value pairs, where each key is a year, retrieve the most recent year @@ -52,7 +73,40 @@ export function toTabularJsonData(jsonData, placeVarName, phsVariableValues) { function getLatestYear(data) { let latestYear = Object.keys(data).sort().reverse()[0]; return parseInt(latestYear); -} +}; + +/** + * Translates a value of an index variable to a DC node identifier (e.g., 94306 -> zip/94306) + * @param value Variable value (e.g., 94306) + * @param variableName Name of the variable (e.g., zip code) + */ +export function indexVariableValueToDcid(variableValue, variableName) { + if (variableValue && variableValue.length > 0) { + if (variableName === INDEX_VARIABLE_NAME_STATE) { + //TODO + } else if (variableName === INDEX_VARIABLE_NAME_COUNTY) { + //TODO + } else if (variableName === INDEX_VARIABLE_NAME_CITY) { + //TODO + } else if (variableName === INDEX_VARIABLE_NAME_ZIP_CODE) { + return 'zip/' + variableValue; + } + } +}; + +export function dcidToIndexVariableValue(dcid, variableName) { + if (dcid && dcid.length > 0) { + if (variableName === INDEX_VARIABLE_NAME_STATE) { + //TODO + } else if (variableName === INDEX_VARIABLE_NAME_COUNTY) { + //TODO + } else if (variableName === INDEX_VARIABLE_NAME_CITY) { + //TODO + } else if (variableName === INDEX_VARIABLE_NAME_ZIP_CODE) { + return dcid.replace('zip/', ''); + } + } +};