In [1]:
# DATS 6103 Srilatha Lakka Project - 2
# Purpose of Project
#1. Learning about Data Mining
#2. Learning to extract information from raw data using Python
'''
About the Data
Data, Trends and Maps is an interactive database that provides data on obesity status '''
#Import required libraries
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import plotly
import plotly.plotly as py
import os
import numpy as np
%matplotlib inline
import plotly.plotly as py
from plotly.offline import init_notebook_mode, iplot
init_notebook_mode(connected = True)
from plotly import tools
import warnings
warnings.filterwarnings('ignore')
import seaborn as sns
import math
In [2]:
py.sign_in('slakka','Pf5T4X7egwQ1QcQZhcJM')
In [3]:
#Extracting Obesity data by state for the year 2017
'''
PURPOSE
"Children and youth are getting fatter every decade”. I found this claim slightly unbelievable, so I decided to investigate…
'''
df= ((pd.read_excel('Nutrition_Physical_Activity_and_Obesity_Behavioral_Risk_Factor_Surveillance_System.xlsx', usecols=(0,2,3,5,7,10,18,29))).fillna(0))
df = df.dropna()
df.head()
Out[3]:
In [4]:
'''
CLEAN AND FORMAT DATA:
Get required data to perform analysis
'''
df1 = df[df['Class'] == ("Obesity / Weight Status")]
df2 = df1[df1['Question'] == ("Percent of adults aged 18 years and older who have obesity")]
df3 = df2[df2['StratificationCategory1'] == ("Age (years)")]
df4 = df3[df3['YearStart'] == (2017)]
df5 = df4[df4['Age(years)'] == ("18 - 24")]
#df5 = df4[df4['Stratification1'] == ("18-24")]
df5.head(10)
Out[4]:
In [25]:
'''
Map US Obesity Percentage by state using plotly
Initial I used LocationDesc column which had State Names, but the Map graph did not show the State Names when hovered over
so I used other column (LocationAbbr) that was available in the dataset which had State abbrevations. Hover over the states
we can see State and percentage of Obesity in those states
'''
'''
The darker regions on the map indicate the State (Oklahoma) with highest percentage of Obese Adult population.
The state with minimum obese adult population is Massachusetts.
'''
scale = [[0.0, 'rgb(223,221,228)'], [0.2, 'rgb(199,199,201)'],
[0.4, 'rgb(169,170,201)'], [0.6, 'rgb(139,135,181)'],
[0.8, 'rgb(98,88,158)'], [1.0, 'rgb(63,20,122)']]
def mapper():
data = [dict(type='choropleth',
colorscale=scale,
locations=df5['LocationAbbr'],
z=df5['Data_Value'],
locationmode='USA-states',
text=df5['LocationAbbr'],
hoverinfo='location+z',
marker=dict(line=dict(color='rgb(255,255,255)', width=2)),
colorbar=dict(title='"Obesity By State For The Year 2017" '))]
layout = dict(title='"Obesity By State For The Year 2017" ' + '<br> Hover for value',
geo=dict(scope='USA',
projection=dict(type='albers usa'), showlakes=True,
lakecolor='rgb(95,145,237)'))
fig = dict(data=data, layout=layout)
return py.iplot(fig, validate=False, filename='Nutrition_Physical_Activity_and_Obesity_Behavioral_Risk_Factor_Surveillance_System.xlsx')
In [24]:
mapper()
Out[24]:
In [7]:
#Extracting Obesity Percentage by Top 10 states which have highest percentage for the year 2017
df6 = df5.nlargest(10, ['Data_Value'])
Top10 = df6.drop(['YearStart', 'LocationAbbr','Class','Question','Age(years)','StratificationCategory1'], axis=1).dropna()
Top10.head()
Top10 = Top10.set_index('LocationDesc')
In [8]:
# Creating a bar diagram showing obesity percentage of top states, in the year 2017.
ax1 = Top10.plot( kind='bar', width= .5, figsize = (10,10), fontsize = 14,
title = 'Highest Obesity Percentage by Top 10 States')
plt.legend(loc = 'best')
ax1.set_xlabel('Percentage', fontsize = 20)
ax1.set_ylabel('Obesity by State', fontsize = 20)
plt.show()
In [9]:
#Extracting Obesity data by state for the year 2017
df= ((pd.read_excel('Nutrition_Physical_Activity_and_Obesity_Behavioral_Risk_Factor_Surveillance_System.xlsx', usecols=(0,2,3,5,7,10,18,29,30))).fillna(0))
df = df.dropna()
df.head()
Out[9]:
In [10]:
# Further analysis to find obesity by gender for above top 10 states
#Extracting Obesity data by gender for the year 2017
# Further analysis to find obesity by gender for above top 10 states
df1gen = df[df['Class'] == ("Obesity / Weight Status")]
df2gen = df1gen[df1gen['Question'] == ("Percent of adults aged 18 years and older who have obesity")]
df3gen = df2gen[df2gen['StratificationCategory1'] == ("Gender")]
df4gen = df3gen[df3gen['YearStart'] == (2017)]
#df5 = df4[df4['Stratification1'] == ("18-24")]
df4gen.head()
Out[10]:
In [11]:
df4gentop = df4gen.nlargest(10, ['Data_Value'])
Top10 = df4gentop.drop(['YearStart', 'LocationAbbr','Class','Question','Age(years)','StratificationCategory1'], axis=1).dropna()
#Rename Columns
Top10.rename(columns={'LocationDesc':'State','Data_Value':'Obesity Percentage','Stratification1':'Gender'}, inplace=True)
Top10 = Top10.set_index('State')
Top10.head(10)
Out[11]:
In [12]:
by_state_gen = Top10.groupby(['State','Obesity Percentage','Gender'])
In [13]:
cat_gen_sz = by_state_gen.size().unstack()
cat_gen_sz['total'] = cat_gen_sz.sum(axis=1)
cat_gen_sz = cat_gen_sz.sort_values(by='Female', ascending=True)
cat_gen_sz[['Female', 'total', 'Male']].plot(kind='barh')
Out[13]:
In [14]:
#Extracting Obesity data by state for the year 2017
df= ((pd.read_excel('Nutrition_Physical_Activity_and_Obesity_Behavioral_Risk_Factor_Surveillance_System.xlsx', usecols=(0,2,3,5,7,10,29,30))).fillna(0))
df = df.dropna()
In [15]:
# Further analysis to find obesity by Ethnicity for top 10 states
#Extracting Obesity data by Ethnicity for the year 2017
df1gen = df[df['Class'] == ("Obesity / Weight Status")]
df2gen = df1gen[df1gen['Question'] == ("Percent of adults aged 18 years and older who have obesity")]
df3gen = df2gen[df2gen['StratificationCategory1'] == ("Race/Ethnicity")]
df4gen = df3gen[df3gen['YearStart'] == (2017)]
#df5 = df4[df4['Stratification1'] == ("18-24")]
df4gen.head()
Out[15]:
In [16]:
df4gentop = df4gen.nlargest(10, ['Data_Value'])
Top10 = df4gentop.drop(['YearStart', 'LocationAbbr','Class','Question','StratificationCategory1'], axis=1).dropna()
#Rename Columns
Top10.rename(columns={'LocationDesc':'State','Data_Value':'Obesity Percentage','Stratification1':'Ethnicity'}, inplace=True)
Top10['State & Ethinicity'] = Top10['State'] +'-'+ '('+Top10['Ethnicity']+ ')'
Top10 = Top10.set_index('State & Ethinicity')
Top10.head(10)
Out[16]:
In [17]:
# Creating a bar diagram showing obesity percentage by Ethnicity, in the year 2017.
ax1 = Top10.plot( kind='barh', width= .5, figsize = (10,10), fontsize = 14,
title = 'Highest Obesity Percentage by Ethnicity')
plt.legend(loc = 'best')
ax1.set_xlabel('State', fontsize = 20)
ax1.set_ylabel('Obesity by State & Ethniity', fontsize = 20)
plt.show()
In [18]:
#Extracting Obesity data by state for year 2011-2017
df= ((pd.read_excel('Nutrition_Physical_Activity_and_Obesity_Behavioral_Risk_Factor_Surveillance_System.xlsx', usecols=(0,2,3,10,18,30,31))).fillna(0))
In [19]:
# Further analysis to find obesity by age group
df3gen = df[df['StratificationCategoryId1'] == ("AGEYR")]
#df3gen = df3gen[df3gen['YearStart'] == (2017)]
df3gen.head()
Out[19]:
In [20]:
Top10 = df3gen.drop(['StratificationCategoryId1'], axis=1).dropna()
In [27]:
Top10.head()
Out[27]:
In [29]:
#Rename Columns
Top10.rename(columns={'Data_Value':'Obesity Percentage','YearStart':'Year'}, inplace=True)
Top10.head()
Out[29]:
In [30]:
Top10group = Top10.groupby(['Age(years)','Year'],as_index=False).agg({"Obesity Percentage": "sum"})
Top10group.head()
Out[30]:
In [31]:
'''
If you look at the Top10 table, the first value is a number. This is the index, and Pandas uses the default Excel practice
of having a number as the index. However, we want to change the index to Year. This will make plotting much easier,
since the index is usually plotted as the x axis.
'''
Top10pivot = Top10group.pivot(columns='Age(years)', index='Year', values='Obesity Percentage')
Top10pivot.head()
Out[31]:
In [32]:
# Plot
Top10pivot.plot()
plt.show()
In [33]:
'''
Let’s just plot a small section of the data: 18-24 and grown ups in the age range of 35-44.
Coming back to our original question: Are children and youth getting obese?
Let’s just plot a small section of the data: 18-24, 35-44 and in the age range of 35-44.
'''
plt.close()
# Plot Youth vs adults
Top10pivot['18 - 24'].plot(label="18 - 24")
Top10pivot['35 - 44'].plot(label="35 - 44")
Top10pivot['45 - 54'].plot(label="45 - 54")
plt.legend(loc="upper right")
plt.show()
'''
Who is getting obese?
Age group 18-24 obesity has gone slightly down, but older age group(35-44 and 45-54) are high in obese
'''
Out[33]:
In [34]:
'''
Let’s plot a small section of the data: 55-64 & 65 and older
'''
plt.close()
# Plot adults
Top10pivot['55 - 64'].plot(label="55 - 64")
Top10pivot['65 or older'].plot(label="65 or older")
plt.legend(loc="upper right")
plt.show()
#Seems like both age groups are almost similar in obsese
In [35]:
# Further analysis to find obesity by Education
df3gen = df[df['StratificationCategoryId1'] == ("EDU")]
#df3gen = df3gen[df3gen['YearStart'] == (2017)]
df3gen.head()
Out[35]:
In [36]:
Top10 = df3gen.drop(['StratificationCategoryId1', 'Age(years)','LocationAbbr'], axis=1).dropna()
Top10.head()
Out[36]:
In [37]:
#Rename Columns
Top10.rename(columns={'Data_Value':'Obesity Percentage','YearStart':'Year','Stratification1':'Education'}, inplace=True)
Top10group.head()
Top10group = Top10.groupby(['Education','Year'],as_index=False).agg({"Obesity Percentage": "sum"})
Top10group.head()
Out[37]:
In [38]:
Top10pivot = Top10group.pivot(columns='Education', index='Year', values='Obesity Percentage')
Top10pivot.head(10)
Out[38]:
In [39]:
# Plot
Top10pivot.plot()
plt.show()
# Seems like College graduates are obese, followed by Technical school
In [40]:
plt.close()
# Plot Youth vs adults
#Top10pivot['College graduate'].plot(label="College graduate")
Top10pivot['Some college or technical school'].plot(label="Some college or technical school")
Top10pivot['High school graduate'].plot(label="High school graduate")
plt.legend(loc="upper right")
plt.show()
'''
Seems like College graduates are obese, followed by Technical school in 2016, but in 2017 High school graduates seems
to obese than other levels of education
'''
Out[40]:
In [41]:
# To find overall obesity percentage
df3gen = df[df['StratificationCategoryId1'] == ("OVR")]
df3gen = df3gen[df3gen['YearStart'] == (2017)]
df3gen.head()
Out[41]:
In [42]:
Top10 = df3gen.drop(['StratificationCategoryId1', 'Age(years)','Stratification1'], axis=1).dropna()
Top10.head()
Out[42]:
In [43]:
#Rename Columns
df4gentop = Top10.nlargest(10, ['Data_Value'])
df4gentop.rename(columns={'Data_Value':'Obesity Percentage','YearStart':'Year','LocationDesc':'State'}, inplace=True)
df4gentop.head()
df4gentopgroup = df4gentop.groupby(['State','Year'],as_index=False).agg({"Obesity Percentage": "sum"})
df4gentopgroup.head(10)
Out[43]:
In [44]:
Top10pivot = df4gentopgroup.pivot(columns='Year', index='State', values='Obesity Percentage')
Top10pivot.head(50)
Out[44]:
In [45]:
# Creating a bar diagram showing obesity percentage by Ethnicity, in the year 2017.
ax1 = Top10pivot.plot( kind='bar', width= .5, figsize = (10,10), fontsize = 10,
title = 'Total Highest Obesity Percentage by Top 10 States for the Year 2017')
plt.legend(loc = 'best')
ax1.set_ylabel('Obesity Percentage', fontsize = 20)
ax1.set_xlabel('Total Obesity by State', fontsize = 20)
plt.show()
In [46]:
# Further analysis to find obesity by age group
df3gen = df[df['StratificationCategoryId1'] == ("AGEYR")]
#df3gen = df3gen[df3gen['YearStart'] == (2017)]
df3gen.head()
Out[46]:
In [ ]:
'''
The graph still doesn’t tell us what will happen to children’s obesity in the future. There are ways to extrapolate graphs
like these into the future, but I must give a warning before we proceed: The obesity data has no underlying mathematical
foundation. That is, we can’t find a formula that will predict how these values will change in the future. Everything is
essentially guesswork. With this warning in mind, let’s see how we can try to extrapolate our graph.
We can try curve fitting:
Curve Fitting tries to fit a curve through points on a graph, by trying to generate a mathematical function for the data.
The function may or may not be very accurate, depending on the data. Polynomial Interpolation Once you have an equation,
you can use polynomial interpolation to try and interpolate any value on the graph.
We’ll use these two functions together to try and predict the future for obesity in youth
'''
In [47]:
Top10 = df3gen.drop(['StratificationCategoryId1'], axis=1).dropna()
Top10.head()
Out[47]:
In [48]:
#Rename Columns
Top10.rename(columns={'Data_Value':'Obesity Percentage','YearStart':'Year'}, inplace=True)
Top10.head()
Out[48]:
In [49]:
Top10group = Top10.groupby(['Age(years)','Year'],as_index=False).agg({"Obesity Percentage": "sum"})
Top10group.head()
Out[49]:
In [50]:
Top10pivot = Top10group.pivot(columns='Age(years)', index='Year', values='Obesity Percentage')
Top10pivot.head()
Out[50]:
In [52]:
youth_obesity_values = Top10pivot['18 - 24'].values
x_axis = range(len(youth_obesity_values))
In [53]:
'''
We set the polynomial degree to 3. We then use the Numpy polyfit() function to try to fit a graph through the data we have.
The poly1d() function is then called on the equation we generated to create a function that will be used to generate our values.
This returns a function called poly_interp that we will use below:
'''
poly_degree = 3
curve_fit = np.polyfit(x_axis, youth_obesity_values, poly_degree)
poly_interp = np.poly1d(curve_fit)
In [54]:
poly_fit_values = []
for i in range(len(x_axis)):
poly_fit_values.append(poly_interp(i))
In [55]:
'''
We will plot both the original data, and our own data, to see how close our equation reached the ideal data
The original data will be plotted in blue and labelled Orig, while the generated data will be red and labelled Fitted.
With a polynomial value of 3:
'''
plt.plot(x_axis, poly_fit_values, "-r", label = "Fitted")
plt.plot(x_axis, youth_obesity_values, "-b", label = "Orig")
plt.legend(loc="upper right")
Out[55]:
In [56]:
#We see it isn’t that good a fit, so let’s try 5:
poly_degree = 5
curve_fit = np.polyfit(x_axis, youth_obesity_values, poly_degree)
poly_interp = np.poly1d(curve_fit)
In [57]:
poly_fit_values = []
for i in range(len(x_axis)):
poly_fit_values.append(poly_interp(i))
In [58]:
plt.plot(x_axis, poly_fit_values, "-r", label = "Fitted")
plt.plot(x_axis, youth_obesity_values, "-b", label = "Orig")
plt.legend(loc="upper right")
Out[58]:
In [ ]:
#We see it isn’t that good a fit, so let’s try 7:
poly_degree = 7
curve_fit = np.polyfit(x_axis, youth_obesity_values, poly_degree)
poly_interp = np.poly1d(curve_fit)
In [59]:
poly_fit_values = []
for i in range(len(x_axis)):
poly_fit_values.append(poly_interp(i))
In [60]:
plt.plot(x_axis, poly_fit_values, "-r", label = "Fitted")
plt.plot(x_axis, youth_obesity_values, "-b", label = "Orig")
plt.legend(loc="upper right")
'''
Now we get an almost perfect match. So, why wouldn’t we always use higher values?
Because the higher values have been so tightly coupled to this graph, they make prediction useless.
'''
Out[60]:
In [67]:
In [ ]: