In [1]:
import pandas as pd
from matplotlib import pyplot as plt
import numpy as np
from lifelines import KaplanMeierFitter
from lifelines import CoxPHFitter
from lifelines.statistics import logrank_test
from lifelines import NelsonAalenFitter
In [2]:
plt.rcParams['figure.figsize'] = [12, 6]
In [3]:
df = pd.read_csv('Outputs/CSV/df_for_statistical_analysis.csv')
Data Prep for Statistical Analysis¶
In [4]:
years = df.loc[:,'1976':'2020']
years
df['Entry'] = years.min(axis=1)
In [5]:
df.dropna(subset=['Entry'], inplace=True)
In [6]:
# Replace empty cells in trainee prog with 0
df['Trainee prog'] = df['Trainee prog'].fillna(0)
In [7]:
# replace dates with 1 in trainee col
df.loc[df['Trainee prog'] > 0, 'Trainee prog'] = 1
In [8]:
# Replace gender col values with 0, 1, 2
df.loc[df['Gender']== 'Male', 'Gender'] = 0
df.loc[df['Gender']== 'Female', 'Gender'] = 1
df.loc[df['Gender']== 'Unknown', 'Gender'] = 2
In [9]:
# Convert consistent role
# 1 = True
# 0 = False
# 2 = Ambiguous: producer/director situation
df.loc[df['Has Consistent Role'] == True, 'Has Consistent Role'] = 1
df.loc[df['Has Consistent Role'] == False, 'Has Consistent Role'] = 2
df.loc[df['Has Consistent Role'].isnull(), 'Has Consistent Role'] = 0
In [10]:
df['Has Consistent Role'].value_counts()
Out[10]:
In [11]:
df.loc[df['Rural'] == True, 'Rural'] = 1
df.loc[df['Rural'].isnull(), 'Rural'] = 0
In [12]:
dept_list = df['Role 1 Category'].value_counts().index.tolist()
In [13]:
# create distinct dfs for each dept
for i in dept_list:
filt = (df['Role 1 Category'] == i)
df_export = df[filt]
name = i.replace(' ', '_')
df_export.to_csv(f'Stats/{i}_df.csv', index=False)
In [14]:
# Assign integers for role cat values
#df.loc[df['Role 1 Category'] == 'Production', 'Role 1 Category'] = 0
for index, val in enumerate(dept_list):
df.loc[df['Role 1 Category'] == val, 'Role 1 Category'] = index
In [15]:
#df['Role 1 Category'] = df['Role 1 Category'].dropna()
df.dropna(subset=['Role 1 Category'], inplace=True)
In [16]:
df['Role 1 Category'].isnull().value_counts()
Out[16]:
In [17]:
# drop useless cols
#df = df.drop(['Role 1 Category', 'Role 2 Category', 'Role 3 Category'], axis=1)
df = df.drop(df.loc[:,'1976':'2020'].columns, axis = 1)
Analysis¶
In [18]:
df['Gender'].hist()
plt.title("Gender")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/gender_hist.png", facecolor='#ffffff')
In [19]:
df['Rural'].hist()
plt.title("Rural")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/rural_hist.png", facecolor='#ffffff')
In [20]:
df.columns
Out[20]:
In [21]:
df['Has Consistent Role'].hist()
plt.title("Role Consistency")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/consistent_hist.png", facecolor='#ffffff')
In [22]:
df['Trainee prog'].hist()
plt.title("Trainee Program")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/trainee_hist.png", facecolor='#ffffff')
In [23]:
df['Trainee prog'].value_counts()
Out[23]:
In [24]:
df.columns
Out[24]:
In [25]:
df['Role 1 Category'].hist()
plt.title("Role 1 Category")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/role_hist.png", facecolor='#ffffff')
In [26]:
#Year of entry to Film Bang
df['Entry'].hist()
plt.title("Entry")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/entry_hist.png", facecolor='#ffffff')
Kaplan-Meier Estimage for Entire Dataset¶
Investigating the longevity of freelancers in the Film Bang database. Longevity here concerns the number of years a freelancer listed in the directry. The birth event is their first entry in the database, and the death event is the retirement of that individual from the directory. Censoring can occur if they are a) still in the directory at the time of dataset compilation (2020).
In [27]:
kmf = KaplanMeierFitter()
In [28]:
df.loc[df['dropout year'] == 0.0, 'dead'] = 0
df.loc[df['dropout year'] > 0, 'dead'] = 1
In [29]:
# drop null values from No of Yrs col
df = df[df['No of Yrs'].notna()]
In [30]:
kmf.fit(durations = df['No of Yrs'],event_observed = df['dead'], label="Film Bang Listings")
Out[30]:
In [31]:
kmf.event_table
Out[31]:
In [32]:
kmf.survival_function_
Out[32]:
In [33]:
kmf.confidence_interval_
Out[33]:
In [34]:
kmf.median_survival_time_
Out[34]:
Kaplan-Meier Plot¶
In [35]:
kmf.plot_survival_function(at_risk_counts=True, label="Listings in Film Bang")
plt.title("Kaplan-Meier Estimate")
# plt.ylabel("Probability of freelancer still in FB")
# plt.xlabel("Years")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/kaplan_meier_estimate_fb_all.png", facecolor='#ffffff')
plt.show
Out[35]:
The y-axis represents the probability a freelancer is still around after 𝑡t years, where 𝑡t years is on the x-axis. We see that very few freelancers make it past 20 years in the listings.
In [36]:
kmf.confidence_interval_
Out[36]:
In [37]:
# Probability of leaving:
kmf.cumulative_density_
Out[37]:
In [38]:
kmf.plot_cumulative_density()
plt.tight_layout()
plt.savefig(f"Stats/Outputs/kaplan_meier_cumulative_density_fb_all.png", facecolor='#ffffff')
In [39]:
# Hazard Function
In [40]:
naf = NelsonAalenFitter()
naf.fit(df['No of Yrs'], event_observed = df['dead'], label="Film Bang Listings")
Out[40]:
In [41]:
naf.cumulative_hazard_
Out[41]:
In [42]:
naf.plot_cumulative_hazard()
plt.tight_layout()
plt.savefig(f"Stats/Outputs/naf_cumulative_hazard_fb_all.png", facecolor='#ffffff')
In [43]:
df['Role 1 Category'].isnull().value_counts()
Out[43]:
In [44]:
# TO DO: REMOVE PARAMETERS WITH MORE THAN 2 VARIABLES - ENTRY AND ROLE
data = df[['No of Yrs', 'Entry','Role 1 Category','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
In [45]:
cph = CoxPHFitter()
cph.fit(data, 'No of Yrs',event_col='dead')
cph.print_summary()
In [46]:
cph.plot()
plt.tight_layout()
plt.savefig(f"Stats/Outputs/cph_plot_fb_all.png", facecolor='#ffffff')
HR greater than 1 means that as the vlaue of the covariate increases, the event hazard increases HR = 1 : no effect HR < 1 : Reduction in the hazard HR > 1 : Increase in hazard
In [47]:
# survival probability for different people in our dataset
d_data = data.iloc[[0,3,4,6]]
cph.predict_survival_function(d_data).plot()
plt.tight_layout()
plt.savefig(f"Stats/Outputs/cph_survival_prediction_4_individuals_fb_all.png", facecolor='#ffffff')
In [48]:
# median time to event for timeline
CTE = kmf.conditional_time_to_event_
plt.plot(CTE)
plt.tight_layout()
plt.savefig(f"Stats/Outputs/kmf_conditional_time_to_event_medians_fb_all.png", facecolor='#ffffff')
Analysis of above chart:¶
Not sure yet...
Gender - Kaplan-Meier Estimate¶
In [49]:
# create 3 kmf objects
kmf_m = KaplanMeierFitter()
kmf_f = KaplanMeierFitter()
kmf_u = KaplanMeierFitter()
In [50]:
Male = df.query("Gender == 0")
Female = df.query("Gender == 1")
Unknown = df.query("Gender == 2")
In [51]:
kmf_m.fit(durations = Male["No of Yrs"],event_observed = Male['dead'],label="Male")
kmf_f.fit(durations = Female["No of Yrs"],event_observed = Female['dead'],label="Female")
kmf_u.fit(durations = Unknown["No of Yrs"],event_observed = Unknown['dead'],label="Unknown")
Out[51]:
In [52]:
kmf_m.event_table
Out[52]:
In [53]:
kmf_f.event_table
Out[53]:
In [54]:
kmf_u.event_table
Out[54]:
In [55]:
kmf_m.survival_function_
Out[55]:
In [56]:
kmf_f.survival_function_
Out[56]:
In [57]:
kmf_u.survival_function_
Out[57]:
In [58]:
kmf_m.plot()
kmf_f.plot()
kmf_u.plot()
plt.xlabel("Years Passed")
plt.ylabel("Longevity")
plt.title("Film Bang - Gender KMF")
plt.tight_layout()
plt.savefig(f"Stats/Outputs/kaplan_meier_fb_all_gender.png", facecolor='#ffffff')
In [59]:
kmf_m.plot_survival_function(at_risk_counts=True, label="Male")
plt.xlabel("Years Passed")
plt.ylabel("Longevity")
plt.title("Film Bang - Gender KMF")
plt.tight_layout()
plt.savefig("Stats/Outputs/kaplan_meier_fb_gender_male.png", facecolor='#ffffff')
In [60]:
kmf_f.plot_survival_function(at_risk_counts=True, label="Female")
plt.xlabel("Years Passed")
plt.ylabel("Longevity")
plt.title("Film Bang - Gender KMF")
plt.tight_layout()
plt.savefig("Stats/Outputs/kaplan_meier_fb_gender_female.png", facecolor='#ffffff')
In [61]:
kmf_u.plot_survival_function(at_risk_counts=True, label="Gender Unknown")
plt.xlabel("Years Passed")
plt.ylabel("Longevity")
plt.title("Film Bang - Gender KMF")
plt.tight_layout()
plt.savefig("Stats/Outputs/kaplan_meier_fb_gender_unknown.png", facecolor='#ffffff')
In [62]:
kmf_m.cumulative_density_
Out[62]:
In [63]:
kmf_f.cumulative_density_
Out[63]:
In [64]:
kmf_u.cumulative_density_
Out[64]:
In [65]:
# Gender Cumulative Density Plot
kmf_m.plot_cumulative_density()
kmf_f.plot_cumulative_density()
kmf_u.plot_cumulative_density()
plt.tight_layout()
plt.savefig("Stats/Outputs/kaplan_meier_cumulative_density_gender.png", facecolor='#ffffff')
Gender Hazard Function¶
In [66]:
naf_m = NelsonAalenFitter()
naf_f = NelsonAalenFitter()
naf_u = NelsonAalenFitter()
naf_m.fit(Male['No of Yrs'],event_observed = Male['dead'], label="Male")
naf_f.fit(Female['No of Yrs'],event_observed = Female['dead'],label="Female")
naf_u.fit(Unknown['No of Yrs'],event_observed = Unknown['dead'],label="Unknown")
Out[66]:
In [67]:
naf_m.cumulative_hazard_
Out[67]:
In [68]:
naf_m.plot_cumulative_hazard()
naf_f.plot_cumulative_hazard()
naf_u.plot_cumulative_hazard()
plt.tight_layout()
plt.savefig("Stats/Outputs/naf_cumulative_hazard_gender.png", facecolor='#ffffff')
Gender - Log Rank Test¶
In [69]:
T1=Male['No of Yrs']
E1=Male['dead']
T2=Female['No of Yrs']
E2=Female['dead']
# T3=Unknown['No of Yrs']
# E3=Unknown['dead']
In [70]:
T1.describe()
Out[70]:
In [71]:
T2.describe()
Out[71]:
In [72]:
results=logrank_test(T1,T2,event_observed_A=E1, event_observed_B=E2)
results.print_summary()
In [73]:
# p value of <0.005 means gender is associated with longevity.
Has Consistent Role¶
In [74]:
kmf_is = KaplanMeierFitter()
kmf_isnt = KaplanMeierFitter()
kmf_dp = KaplanMeierFitter()
In [75]:
df = df.rename(columns={"Has Consistent Role": "Consistent"})
In [76]:
df['Consistent']
Out[76]:
In [77]:
Consistent = df.query("Consistent == 1")
Inconsistent = df.query("Consistent == 2")
DirProd = df.query("Consistent == 0")
In [78]:
kmf_is.fit(durations = Consistent['No of Yrs'],event_observed= Consistent['dead'], label="Consistent")
kmf_isnt.fit(durations = Inconsistent['No of Yrs'],event_observed= Inconsistent['dead'], label="Inconsistent")
kmf_dp.fit(durations = DirProd['No of Yrs'],event_observed= DirProd['dead'], label="Director/Producer")
Out[78]:
In [79]:
kmf_is.event_table
Out[79]:
In [80]:
kmf_isnt.event_table
Out[80]:
In [81]:
kmf_dp.event_table
Out[81]:
In [82]:
kmf_is.survival_function_
Out[82]:
In [83]:
kmf_isnt.survival_function_
Out[83]:
In [84]:
kmf_dp.survival_function_
Out[84]:
In [85]:
kmf_is.plot()
kmf_isnt.plot()
kmf_dp.plot()
plt.xlabel("Years")
plt.ylabel("Longevity")
plt.title("Film Bang - Role Consistency KMF")
plt.tight_layout()
plt.savefig("Stats/Outputs/kapplan_meier_estimate_role_consistency_all.png", facecolor='#ffffff')
In [86]:
kmf_is.plot_cumulative_density()
kmf_isnt.plot_cumulative_density()
kmf_dp.plot_cumulative_density()
plt.tight_layout()
plt.savefig("Stats/Outputs/kaplan_meier_cumulative_density_role_consistency_all.png", facecolor='#ffffff')
In [87]:
naf_is = NelsonAalenFitter()
naf_isnt = NelsonAalenFitter()
naf_dp = NelsonAalenFitter()
naf_is.fit(Consistent['No of Yrs'],event_observed = Consistent['dead'])
naf_isnt.fit(Inconsistent['No of Yrs'],event_observed = Inconsistent['dead'])
naf_dp.fit(DirProd['No of Yrs'],event_observed = DirProd['dead'])
Out[87]:
In [88]:
naf_is.plot_cumulative_hazard(label="Consistent")
naf_isnt.plot_cumulative_hazard(label="Inconsistent")
naf_dp.plot_cumulative_hazard(label="Director/Producer")
plt.tight_layout()
plt.savefig("Stats/Outputs/naf_cumulative_hazard_role_consistency.png", facecolor='#ffffff')
Consistent Role Log-Rank Test¶
In [89]:
T1=Consistent['No of Yrs']
T2=Inconsistent['No of Yrs']
E1=Consistent['dead']
E2=Inconsistent['dead']
In [90]:
results=logrank_test(T1,T2,event_observed_A=E1, event_observed_B=E2)
results.print_summary()
Trainee Prog¶
In [91]:
kmf_t = KaplanMeierFitter()
kmf_null = KaplanMeierFitter()
In [92]:
df = df.rename(columns={"Trainee prog": "Trainee"})
In [93]:
Trainee = df.query("Trainee == 1")
Non = df.query("Trainee == 0")
In [94]:
kmf_t.fit(durations = Trainee['No of Yrs'],event_observed= Trainee['dead'], label="Trainee")
kmf_null.fit(durations = Non['No of Yrs'],event_observed= Non['dead'], label="Non Trainee")
Out[94]:
In [95]:
kmf_t.event_table
Out[95]:
In [96]:
kmf_null.event_table
Out[96]:
In [97]:
kmf_t.survival_function_
Out[97]:
In [98]:
kmf_null.survival_function_
Out[98]:
In [99]:
kmf_t.plot()
kmf_null.plot()
plt.xlabel("Years")
plt.ylabel("Longevity")
plt.title("Film Bang - Trainee Prog KMF")
plt.tight_layout()
plt.savefig("Stats/Outputs/kaplan_meier_estimate_trainee.png", facecolor='#ffffff')
In [100]:
kmf_t.plot_cumulative_density()
kmf_null.plot_cumulative_density()
plt.tight_layout()
plt.savefig("Stats/Outputs/kaplan_meier_cumulative_density_trainee.png", facecolor='#ffffff')
In [101]:
naf_t = NelsonAalenFitter()
naf_null = NelsonAalenFitter()
naf_t.fit(Trainee['No of Yrs'],event_observed = Trainee['dead'])
naf_null.fit(Non['No of Yrs'],event_observed = Non['dead'])
Out[101]:
In [102]:
naf_t.plot_cumulative_hazard(label="Trainee")
naf_null.plot_cumulative_hazard(label="Non Trainee")
plt.tight_layout()
plt.savefig("Stats/Outputs/naf_cumulative_hazard_trainee.png", facecolor='#ffffff')
Trainee Log-Rank Test¶
In [103]:
T1=Trainee['No of Yrs']
T2=Non['No of Yrs']
E1=Trainee['dead']
E2=Non['dead']
In [104]:
T1.describe()
Out[104]:
In [105]:
T2.describe()
Out[105]:
In [106]:
results=logrank_test(T1,T2,event_observed_A=E1, event_observed_B=E2)
results.print_summary()
Depts¶
In [107]:
#df.columns
In [108]:
#dept_list
Production¶
In [109]:
production = pd.read_csv('Stats/Production_df.csv')
In [110]:
kmf_production = KaplanMeierFitter()
In [111]:
production.shape
Out[111]:
In [112]:
production.loc[production['dropout year'] == 0.0, 'dead'] = 0
production.loc[production['dropout year'] > 0, 'dead'] = 1
In [113]:
# drop null values from No of Yrs col
production = production[production['No of Yrs'].notna()]
In [114]:
kmf_production.fit(durations = production['No of Yrs'],event_observed = production['dead'], label="Production")
Out[114]:
In [115]:
kmf_production.event_table
Out[115]:
In [116]:
kmf_production.survival_function_
Out[116]:
In [117]:
kmf_production.median_survival_time_
Out[117]:
In [118]:
kmf_production.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[118]:
In [119]:
kmf_production.confidence_interval_
Out[119]:
In [120]:
# Probability of leaving:
kmf_production.cumulative_density_
Out[120]:
In [121]:
kmf_production.plot_cumulative_density()
Out[121]:
In [122]:
naf_production = NelsonAalenFitter()
naf_production.fit(production['No of Yrs'], event_observed = production['dead'])
Out[122]:
In [123]:
naf.cumulative_hazard_
Out[123]:
In [124]:
naf_production.plot_cumulative_hazard()
Out[124]:
In [125]:
data_production = production[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_production = CoxPHFitter()
cph_production.fit(data_production, 'No of Yrs',event_col='dead')
cph_production.print_summary()
Camera¶
In [126]:
camera = pd.read_csv('Stats/Camera_df.csv')
In [127]:
kmf_camera = KaplanMeierFitter()
In [128]:
camera.shape
Out[128]:
In [129]:
camera.loc[camera['dropout year'] == 0.0, 'dead'] = 0
camera.loc[camera['dropout year'] > 0, 'dead'] = 1
In [130]:
# drop null values from No of Yrs col
camera = camera[camera['No of Yrs'].notna()]
In [131]:
kmf_camera.fit(durations = camera['No of Yrs'],event_observed = camera['dead'], label="Camera")
Out[131]:
In [132]:
kmf_production.event_table
Out[132]:
In [133]:
kmf_production.survival_function_
Out[133]:
In [134]:
kmf_camera.median_survival_time_
Out[134]:
In [135]:
kmf_camera.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[135]:
In [136]:
kmf_production.confidence_interval_
Out[136]:
In [137]:
# Probability of leaving:
kmf_production.cumulative_density_
Out[137]:
In [138]:
kmf_camera.plot_cumulative_density()
Out[138]:
In [139]:
naf_camera = NelsonAalenFitter()
naf_camera.fit(camera['No of Yrs'], event_observed = camera['dead'], label="Camera")
Out[139]:
In [140]:
naf.cumulative_hazard_
Out[140]:
In [141]:
naf_camera.plot_cumulative_hazard()
Out[141]:
In [142]:
camera = camera[camera['Entry'].notna()]
In [143]:
data_camera = camera[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_camera = CoxPHFitter()
cph_camera.fit(data_camera, 'No of Yrs',event_col='dead')
cph_camera.print_summary()
Art¶
In [144]:
art = pd.read_csv('Stats/Art_df.csv')
In [145]:
kmf_art = KaplanMeierFitter()
In [146]:
art.shape
Out[146]:
In [147]:
art.loc[art['dropout year'] == 0.0, 'dead'] = 0
art.loc[art['dropout year'] > 0, 'dead'] = 1
In [148]:
# drop null values from No of Yrs col
art = art[art['No of Yrs'].notna()]
In [149]:
kmf_art.fit(durations = art['No of Yrs'],event_observed = art['dead'], label="Art")
Out[149]:
In [150]:
kmf_production.event_table
Out[150]:
In [151]:
kmf_production.survival_function_
Out[151]:
In [152]:
kmf_art.median_survival_time_
Out[152]:
In [153]:
kmf_art.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[153]:
In [154]:
kmf_production.confidence_interval_
Out[154]:
In [155]:
# Probability of leaving:
kmf_production.cumulative_density_
Out[155]:
In [156]:
kmf_art.plot_cumulative_density()
Out[156]:
In [157]:
naf_art = NelsonAalenFitter()
naf_art.fit(df['No of Yrs'], event_observed = df['dead'], label="Art")
Out[157]:
In [158]:
naf.cumulative_hazard_
Out[158]:
In [159]:
naf_art.plot_cumulative_hazard()
Out[159]:
In [160]:
art = art[art['Entry'].notna()]
In [161]:
data_art = art[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_art = CoxPHFitter()
cph_art.fit(data_art, 'No of Yrs',event_col='dead')
cph_art.print_summary()
Direction¶
In [162]:
direction = pd.read_csv('Stats/Direction_df.csv')
In [163]:
kmf_direction = KaplanMeierFitter()
In [164]:
direction.shape
Out[164]:
In [165]:
direction.loc[direction['dropout year'] == 0.0, 'dead'] = 0
direction.loc[direction['dropout year'] > 0, 'dead'] = 1
In [166]:
# drop null values from No of Yrs col
direction = direction[direction['No of Yrs'].notna()]
In [167]:
kmf_direction.fit(durations = direction['No of Yrs'],event_observed = direction['dead'], label="Direction")
Out[167]:
In [168]:
kmf_production.event_table
Out[168]:
In [169]:
kmf_production.survival_function_
Out[169]:
In [170]:
kmf_direction.median_survival_time_
Out[170]:
In [171]:
kmf_direction.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[171]:
In [172]:
kmf_production.confidence_interval_
Out[172]:
In [173]:
# Probability of leaving:
kmf_production.cumulative_density_
Out[173]:
In [174]:
kmf_direction.plot_cumulative_density()
Out[174]:
In [175]:
naf_direction = NelsonAalenFitter()
naf_direction.fit(direction['No of Yrs'], event_observed = direction['dead'], label="Direction")
Out[175]:
In [176]:
naf.cumulative_hazard_
Out[176]:
In [177]:
naf_direction.plot_cumulative_hazard()
Out[177]:
In [178]:
data_direction = direction[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_direction = CoxPHFitter()
cph_direction.fit(data_direction, 'No of Yrs',event_col='dead')
cph_direction.print_summary()
Producer¶
In [179]:
producer = pd.read_csv('Stats/Producer_df.csv')
In [180]:
kmf_producer = KaplanMeierFitter()
In [181]:
producer.shape
Out[181]:
In [182]:
producer.loc[producer['dropout year'] == 0.0, 'dead'] = 0
producer.loc[producer['dropout year'] > 0, 'dead'] = 1
In [183]:
# drop null values from No of Yrs col
producer = producer[producer['No of Yrs'].notna()]
In [184]:
kmf_producer.fit(durations = producer['No of Yrs'],event_observed = producer['dead'], label="Producer")
Out[184]:
In [185]:
kmf_production.event_table
Out[185]:
In [186]:
kmf_production.survival_function_
Out[186]:
In [187]:
kmf_producer.median_survival_time_
Out[187]:
In [188]:
kmf_producer.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[188]:
In [189]:
kmf_production.confidence_interval_
Out[189]:
In [190]:
# Probability of leaving:
kmf_production.cumulative_density_
Out[190]:
In [191]:
kmf_producer.plot_cumulative_density()
Out[191]:
In [192]:
naf_producer = NelsonAalenFitter()
naf_producer.fit(producer['No of Yrs'], event_observed = producer['dead'], label="Producer")
Out[192]:
In [193]:
naf.cumulative_hazard_
Out[193]:
In [194]:
naf_producer.plot_cumulative_hazard()
Out[194]:
In [195]:
data_producer = producer[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_producer = CoxPHFitter()
cph_producer.fit(data_producer, 'No of Yrs',event_col='dead')
cph_producer.print_summary()
Post-Production¶
In [196]:
p_production = pd.read_csv('Stats/Post-Production_df.csv')
In [197]:
kmf_p_production = KaplanMeierFitter()
In [198]:
p_production.shape
Out[198]:
In [199]:
p_production.loc[p_production['dropout year'] == 0.0, 'dead'] = 0
p_production.loc[p_production['dropout year'] > 0, 'dead'] = 1
In [200]:
# drop null values from No of Yrs col
p_production = p_production[p_production['No of Yrs'].notna()]
In [201]:
kmf_p_production.fit(durations = p_production['No of Yrs'],event_observed = p_production['dead'], label="Post-Production")
Out[201]:
In [202]:
kmf_production.event_table
Out[202]:
In [203]:
kmf_production.survival_function_
Out[203]:
In [204]:
kmf_p_production.median_survival_time_
Out[204]:
In [205]:
kmf_p_production.confidence_interval_
Out[205]:
In [206]:
kmf_p_production.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Post-Production freelancer still in FB")
plt.show
Out[206]:
In [207]:
kmf_p_production.confidence_interval_
Out[207]:
In [208]:
# Probability of leaving:
kmf_p_production.cumulative_density_
Out[208]:
In [209]:
kmf_p_production.plot_cumulative_density()
Out[209]:
In [210]:
naf_p_production = NelsonAalenFitter()
naf_p_production.fit(p_production['No of Yrs'], event_observed = p_production['dead'], label="Post-Production")
Out[210]:
In [211]:
naf.cumulative_hazard_
Out[211]:
In [212]:
naf_p_production.plot_cumulative_hazard()
Out[212]:
In [213]:
data_p_production = p_production[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_p_production = CoxPHFitter()
cph_p_production.fit(data_p_production, 'No of Yrs',event_col='dead')
cph_p_production.print_summary()
Sound¶
In [214]:
sound = pd.read_csv('Stats/Sound_df.csv')
In [215]:
kmf_sound = KaplanMeierFitter()
In [216]:
sound.shape
Out[216]:
In [217]:
sound.loc[sound['dropout year'] == 0.0, 'dead'] = 0
sound.loc[sound['dropout year'] > 0, 'dead'] = 1
In [218]:
# drop null values from No of Yrs col
sound = sound[sound['No of Yrs'].notna()]
In [219]:
kmf_sound.fit(durations = sound['No of Yrs'],event_observed = sound['dead'], label="Sound")
Out[219]:
In [220]:
kmf_production.event_table
Out[220]:
In [221]:
kmf_production.survival_function_
Out[221]:
In [222]:
kmf_sound.median_survival_time_
Out[222]:
In [223]:
kmf_p_production.confidence_interval_
Out[223]:
In [224]:
kmf_sound.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[224]:
In [225]:
kmf_sound.confidence_interval_
Out[225]:
In [226]:
# Probability of leaving:
kmf_sound.cumulative_density_
Out[226]:
In [227]:
kmf_sound.plot_cumulative_density()
Out[227]:
In [228]:
naf_sound = NelsonAalenFitter()
naf_sound.fit(sound['No of Yrs'], event_observed = sound['dead'], label="Sound")
Out[228]:
In [229]:
naf.cumulative_hazard_
Out[229]:
In [230]:
naf_sound.plot_cumulative_hazard()
Out[230]:
In [231]:
data_sound = sound[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_sound = CoxPHFitter()
cph_sound.fit(data_sound, 'No of Yrs',event_col='dead')
cph_sound.print_summary()
Hair & Make-Up¶
In [232]:
hair_make_up = pd.read_csv('Stats/Hair_&_Make-Up_df.csv')
In [233]:
kmf_hair_make_up = KaplanMeierFitter()
In [234]:
hair_make_up.shape
Out[234]:
In [235]:
hair_make_up.loc[hair_make_up['dropout year'] == 0.0, 'dead'] = 0
hair_make_up.loc[hair_make_up['dropout year'] > 0, 'dead'] = 1
In [236]:
# drop null values from No of Yrs col
p_prodhair_make_upuction = hair_make_up[hair_make_up['No of Yrs'].notna()]
In [237]:
kmf_hair_make_up.fit(durations = hair_make_up['No of Yrs'],event_observed = hair_make_up['dead'], label="Hair & Make-Up")
Out[237]:
In [238]:
kmf_production.event_table
Out[238]:
In [239]:
kmf_production.survival_function_
Out[239]:
In [240]:
kmf_hair_make_up.median_survival_time_
Out[240]:
In [241]:
kmf_p_production.confidence_interval_
Out[241]:
In [242]:
kmf_hair_make_up.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[242]:
In [243]:
kmf_hair_make_up.confidence_interval_
Out[243]:
In [244]:
# Probability of leaving:
kmf_hair_make_up.cumulative_density_
Out[244]:
In [245]:
kmf_hair_make_up.plot_cumulative_density()
Out[245]:
In [246]:
naf_hair_make_up = NelsonAalenFitter()
naf_hair_make_up.fit(hair_make_up['No of Yrs'], event_observed = hair_make_up['dead'], label="Hair & Make-Up")
Out[246]:
In [247]:
naf.cumulative_hazard_
Out[247]:
In [248]:
naf_hair_make_up.plot_cumulative_hazard()
Out[248]:
In [249]:
data_hair_make_up = hair_make_up[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_hair_make_up = CoxPHFitter()
cph_hair_make_up.fit(data_hair_make_up, 'No of Yrs',event_col='dead')
cph_hair_make_up.print_summary()
Costume¶
In [250]:
costume = pd.read_csv('Stats/Costume_df.csv')
In [251]:
kmf_costume = KaplanMeierFitter()
In [252]:
costume.shape
Out[252]:
In [253]:
costume.loc[costume['dropout year'] == 0.0, 'dead'] = 0
costume.loc[costume['dropout year'] > 0, 'dead'] = 1
In [254]:
# drop null values from No of Yrs col
costume = costume[costume['No of Yrs'].notna()]
In [255]:
kmf_costume.fit(durations = costume['No of Yrs'],event_observed = costume['dead'], label="Costume")
Out[255]:
In [256]:
kmf_production.event_table
Out[256]:
In [257]:
kmf_production.survival_function_
Out[257]:
In [258]:
kmf_costume.median_survival_time_
Out[258]:
In [259]:
kmf_p_production.confidence_interval_
Out[259]:
In [260]:
kmf_costume.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[260]:
In [261]:
kmf_costume.confidence_interval_
Out[261]:
In [262]:
# Probability of leaving:
kmf_costume.cumulative_density_
Out[262]:
In [263]:
kmf_costume.plot_cumulative_density()
Out[263]:
In [264]:
naf_costume = NelsonAalenFitter()
naf_costume.fit(costume['No of Yrs'], event_observed = costume['dead'], label="Costume")
Out[264]:
In [265]:
naf.cumulative_hazard_
Out[265]:
In [266]:
naf_costume.plot_cumulative_hazard()
Out[266]:
In [267]:
data_costume = costume[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_costume = CoxPHFitter()
cph_costume.fit(data_costume, 'No of Yrs',event_col='dead')
cph_costume.print_summary()
In [ ]:
Music¶
In [268]:
music = pd.read_csv('Stats/Music_df.csv')
In [269]:
kmf_music = KaplanMeierFitter()
In [270]:
music.shape
Out[270]:
In [271]:
music.loc[music['dropout year'] == 0.0, 'dead'] = 0
music.loc[music['dropout year'] > 0, 'dead'] = 1
In [272]:
# drop null values from No of Yrs col
music = music[music['No of Yrs'].notna()]
In [273]:
kmf_music.fit(durations = music['No of Yrs'],event_observed = music['dead'], label="Music")
Out[273]:
In [274]:
kmf_music.event_table
Out[274]:
In [275]:
kmf_music.survival_function_
Out[275]:
In [276]:
kmf_music.median_survival_time_
Out[276]:
In [277]:
kmf_p_production.confidence_interval_
Out[277]:
In [278]:
kmf_music.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[278]:
In [279]:
kmf_music.confidence_interval_
Out[279]:
In [280]:
# Probability of leaving:
kmf_music.cumulative_density_
Out[280]:
In [281]:
kmf_music.plot_cumulative_density()
Out[281]:
In [282]:
naf_music = NelsonAalenFitter()
naf_music.fit(music['No of Yrs'], event_observed = music['dead'], label="Music")
Out[282]:
In [283]:
naf.cumulative_hazard_
Out[283]:
In [284]:
naf_music.plot_cumulative_hazard()
Out[284]:
In [285]:
music['No of Yrs'].isnull().value_counts(dropna=False)
Out[285]:
In [286]:
# Trainee Prog col dropped due to convergence issue: very low incidence.
data_music = music[['No of Yrs', 'Entry','Gender','Rural', 'Has Consistent Role','dead']]
cph_music = CoxPHFitter()
cph_music.fit(data_music, 'No of Yrs',event_col='dead')
cph_music.print_summary()
Support¶
In [287]:
support = pd.read_csv('Stats/Support_df.csv')
In [288]:
kmf_support = KaplanMeierFitter()
In [289]:
support.shape
Out[289]:
In [290]:
support.loc[support['dropout year'] == 0.0, 'dead'] = 0
support.loc[support['dropout year'] > 0, 'dead'] = 1
In [291]:
# drop null values from No of Yrs col
support = support[support['No of Yrs'].notna()]
In [292]:
# check
#df['No of Yrs'].value_counts(dropna=False)
In [293]:
kmf_support.fit(durations = support['No of Yrs'],event_observed = support['dead'], label="Support")
Out[293]:
In [294]:
kmf_support.event_table
Out[294]:
In [295]:
kmf_support.survival_function_
Out[295]:
In [296]:
kmf_support.median_survival_time_
Out[296]:
In [297]:
kmf_p_production.confidence_interval_
Out[297]:
In [298]:
kmf_support.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[298]:
In [299]:
kmf_support.confidence_interval_
Out[299]:
In [300]:
# Probability of leaving:
kmf_support.cumulative_density_
Out[300]:
In [301]:
kmf_support.plot_cumulative_density()
Out[301]:
In [302]:
naf_support = NelsonAalenFitter()
naf_support.fit(support['No of Yrs'], event_observed = support['dead'], label="Support")
Out[302]:
In [303]:
naf.cumulative_hazard_
Out[303]:
In [304]:
naf_support.plot_cumulative_hazard()
Out[304]:
In [305]:
data_support = support[['No of Yrs', 'Entry','Trainee prog','Gender','Rural', 'Has Consistent Role','dead']]
cph_support = CoxPHFitter()
cph_support.fit(data_support, 'No of Yrs',event_col='dead')
cph_support.print_summary()
Script¶
In [306]:
script = pd.read_csv('Stats/Script_df.csv')
In [307]:
kmf_script = KaplanMeierFitter()
In [308]:
script.shape
Out[308]:
In [309]:
script.loc[script['dropout year'] == 0.0, 'dead'] = 0
script.loc[script['dropout year'] > 0, 'dead'] = 1
In [310]:
# drop null values from No of Yrs col
script = script[script['No of Yrs'].notna()]
In [311]:
kmf_script.fit(durations = script['No of Yrs'],event_observed = script['dead'], label="Script")
Out[311]:
In [312]:
kmf_script.event_table
Out[312]:
In [313]:
kmf_script.survival_function_
Out[313]:
In [314]:
kmf_script.median_survival_time_
Out[314]:
In [315]:
kmf_p_production.confidence_interval_
Out[315]:
In [316]:
kmf_script.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[316]:
In [317]:
kmf_script.confidence_interval_
Out[317]:
In [318]:
# Probability of leaving:
kmf_script.cumulative_density_
Out[318]:
In [319]:
kmf_script.plot_cumulative_density()
Out[319]:
In [320]:
naf_script = NelsonAalenFitter()
naf_script.fit(script['No of Yrs'], event_observed = script['dead'], label="Script")
Out[320]:
In [321]:
naf.cumulative_hazard_
Out[321]:
In [322]:
naf_script.plot_cumulative_hazard()
Out[322]:
In [323]:
# Trainee prog and Rural cols dropped due to convegece issue: low incidence
data_script = script[['No of Yrs', 'Entry','Gender', 'Has Consistent Role','dead']]
cph_script = CoxPHFitter()
cph_script.fit(data_script, 'No of Yrs',event_col='dead')
cph_script.print_summary()
Casting¶
In [324]:
casting = pd.read_csv('Stats/Casting_df.csv')
In [325]:
kmf_casting = KaplanMeierFitter()
In [326]:
casting.shape
Out[326]:
In [327]:
casting.loc[casting['dropout year'] == 0.0, 'dead'] = 0
casting.loc[casting['dropout year'] > 0, 'dead'] = 1
In [328]:
# drop null values from No of Yrs col
casting = casting[casting['No of Yrs'].notna()]
In [329]:
kmf_casting.fit(durations = casting['No of Yrs'],event_observed = casting['dead'], label="Casting")
Out[329]:
In [330]:
kmf_casting.event_table
Out[330]:
In [331]:
kmf_casting.survival_function_
Out[331]:
In [332]:
kmf_casting.median_survival_time_
Out[332]:
In [333]:
kmf_p_production.confidence_interval_
Out[333]:
In [334]:
kmf_casting.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[334]:
In [335]:
kmf_casting.confidence_interval_
Out[335]:
In [336]:
# Probability of leaving:
kmf_casting.cumulative_density_
Out[336]:
In [337]:
kmf_casting.plot_cumulative_density()
Out[337]:
In [338]:
naf_casting = NelsonAalenFitter()
naf_casting.fit(casting['No of Yrs'], event_observed = casting['dead'], label="Casting")
Out[338]:
In [339]:
naf.cumulative_hazard_
Out[339]:
In [340]:
naf_casting.plot_cumulative_hazard()
Out[340]:
In [341]:
# Trainee prog and Rural columns dropped due to low incidence causing convergence issue
data_casting = casting[['No of Yrs', 'Entry','Gender', 'Has Consistent Role','dead']]
cph_casting = CoxPHFitter()
cph_casting.fit(data_casting, 'No of Yrs',event_col='dead')
cph_casting.print_summary()
Construction¶
In [342]:
construction = pd.read_csv('Stats/Construction_df.csv')
In [343]:
kmf_construction = KaplanMeierFitter()
In [344]:
construction.shape
Out[344]:
In [345]:
construction.loc[construction['dropout year'] == 0.0, 'dead'] = 0
construction.loc[construction['dropout year'] > 0, 'dead'] = 1
In [346]:
# drop null values from No of Yrs col
construction = construction[construction['No of Yrs'].notna()]
In [347]:
kmf_construction.fit(durations = construction['No of Yrs'],event_observed = construction['dead'], label="Construction")
Out[347]:
In [348]:
kmf_construction.event_table
Out[348]:
In [349]:
kmf_construction.survival_function_
Out[349]:
In [350]:
kmf_construction.median_survival_time_
Out[350]:
In [351]:
kmf_construction.confidence_interval_
Out[351]:
In [352]:
kmf_construction.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[352]:
In [353]:
kmf_construction.confidence_interval_
Out[353]:
In [354]:
# Probability of leaving:
kmf_construction.cumulative_density_
Out[354]:
In [355]:
kmf_construction.plot_cumulative_density()
Out[355]:
In [356]:
naf_construction = NelsonAalenFitter()
naf_construction.fit(construction['No of Yrs'], event_observed = construction['dead'], label="Construction")
Out[356]:
In [357]:
naf.cumulative_hazard_
Out[357]:
In [358]:
naf_construction.plot_cumulative_hazard()
Out[358]:
In [359]:
#Rural, Gender, Has Consistent Role, Trainee prog removed due to onvergence issue: low incidence.
data_construction = construction[['No of Yrs', 'Entry','dead']]
cph_construction = CoxPHFitter()
cph_construction.fit(data_construction, 'No of Yrs',event_col='dead')
cph_construction.print_summary()
Special FX¶
In [360]:
special_fx = pd.read_csv('Stats/Special_FX_df.csv')
In [361]:
kmf_special_fx = KaplanMeierFitter()
In [362]:
special_fx.shape
Out[362]:
In [363]:
special_fx.loc[special_fx['dropout year'] == 0.0, 'dead'] = 0
special_fx.loc[special_fx['dropout year'] > 0, 'dead'] = 1
In [364]:
# drop null values from No of Yrs col
special_fx = special_fx[special_fx['No of Yrs'].notna()]
In [365]:
kmf_special_fx.fit(durations = special_fx['No of Yrs'],event_observed = special_fx['dead'], label="Special FX")
Out[365]:
In [366]:
kmf_special_fx.event_table
Out[366]:
In [367]:
kmf_special_fx.survival_function_
Out[367]:
In [368]:
kmf_special_fx.median_survival_time_
Out[368]:
In [369]:
kmf_p_production.confidence_interval_
Out[369]:
In [370]:
kmf_special_fx.plot_survival_function(at_risk_counts=True)
plt.title("Kaplan-Meier Estimate")
plt.ylabel("Probability of Production freelancer still in FB")
plt.show
Out[370]:
In [371]:
kmf_special_fx.confidence_interval_
Out[371]:
In [372]:
# Probability of leaving:
kmf_special_fx.cumulative_density_
Out[372]:
In [373]:
kmf_special_fx.plot_cumulative_density()
Out[373]:
In [374]:
naf_special_fx = NelsonAalenFitter()
naf_special_fx.fit(special_fx['No of Yrs'], event_observed = special_fx['dead'], label="Special FX")
Out[374]:
In [375]:
naf.cumulative_hazard_
Out[375]:
In [376]:
naf_special_fx.plot_cumulative_hazard()
Out[376]:
In [377]:
# Trainee prog col dropped due to convergence issue: low incidence
data_special_fx = special_fx[['No of Yrs', 'Entry','Gender','Rural', 'Has Consistent Role','dead']]
cph_special_fx = CoxPHFitter()
cph_special_fx.fit(data_special_fx, 'No of Yrs',event_col='dead')
cph_special_fx.print_summary()
In [378]:
# Test
fig1 = plt.figure()
ax1 = plt.subplot(211)
ax2 = plt.subplot(211)
ax3 = plt.subplot(211)
ax4 = plt.subplot(211)
ax5 = plt.subplot(211)
ax6 = plt.subplot(211)
ax7 = plt.subplot(211)
ax8 = plt.subplot(211)
ax9 = plt.subplot(211)
ax10 = plt.subplot(211)
ax11 = plt.subplot(211)
ax12 = plt.subplot(211)
ax13 = plt.subplot(211)
ax14 = plt.subplot(211)
ax15 = plt.subplot(211)
kmf_sound.survival_function_.plot(ax=ax7, label="sound")
kmf_camera.survival_function_.plot(ax=ax4, label="camera")
kmf_producer.survival_function_.plot(ax=ax5, label="producer")
kmf_art.survival_function_.plot(ax=ax3, label="art")
kmf_direction.survival_function_.plot(ax=ax2, label="direction")
kmf_p_production.survival_function_.plot(ax=ax6, label="post production")
kmf_hair_make_up.survival_function_.plot(ax=ax8, label="hair & make-up")
kmf_costume.survival_function_.plot(ax=ax9, label="costume")
kmf_script.survival_function_.plot(ax=ax12, label="script")
kmf_support.survival_function_.plot(ax=ax11, label="support")
kmf_casting.survival_function_.plot(ax=ax13, label="casting")
kmf_production.survival_function_.plot(ax=ax1, label="production")
kmf_special_fx.survival_function_.plot(ax=ax15, label="special fx")
kmf_construction.survival_function_.plot(ax=ax14, label="construction")
kmf_music.survival_function_.plot(ax=ax10, label="music")
fig1.set_size_inches(12, 10)
plt.ylabel("Probability of Production freelancer still in FB")
######
plt.tight_layout()
plt.savefig("Stats/Outputs/dept_kmf_estimates_3.png", facecolor='#ffffff')
plt.show
Out[378]:
