1
2
3

df = pd.read_csv('file_path')

df2 = pd.read_table('file_path')

1

chunker = pd.read_csv('train.csv', chunksize=1000)

1
2
3
4
5
6

chunker = pd.read_csv("data.csv",chunksize=5)
    for piece in chunker:
        print(type(piece))
        #<class 'pandas.core.frame.DataFrame'>
        print(len(piece))

1

df = {'Years':[2001,2002,2003,2004,2005],'boold':[true,false,true,true,false],'strd':'1','2','3','4','5'}

1

sdata = {'boold':true,'year':2002,'strd':'Series'}

1
2
3
4
5
6

#对家庭成员进行分析
family = df['堂兄第/妹个数']+df['父母与小孩个数']
df.insert(6,'家庭人数', family)
df.head()
df.sort_values(by=["家庭人数"],ascending = False).head(10)
plt.plot(plttest["家庭人数"],plttest["是否幸存"],'ko--')

1
2

type(np.nan)  # <class 'float'>
type(None)    #<class 'NoneType'>

1

df['Age'].fillna(df['Age'].describe()['mean'], inplace = True)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

import matplotlib.pyplot as plt
x = list(k_Cabin['Cabin'].unique())
y =[]
Cabin_k = np.array([df.Cabin.isnull() & df.Survived == 1])
for i in x:
    tmp = 0
    all_people = k_Cabin['Cabin'][k_Cabin['Cabin'] == i].count()
    arrived_people = k_Cabin['Cabin'][(k_Cabin.Cabin == i) & (k_Cabin.Survived == 1)].count()
    y.append(arrived_people / all_people)
x.append('No Cabin')
y.append(np.sum(Cabin_k !=0) / Cabin_k.shape[1])
plt.plot(x,y)
plt.ylabel('Survived rate')
plt.show()

1
2
3
4

#初步思路
df.Cabin[df.Cabin.isnull() & df.Survived == 1].count()
##得出结果 0     发现数据为NaN时，count()得出结果始终为0

1
2

Cabin_k = np.array([df.Cabin.isnull() & df.Survived == 1])
print(np.sum(Cabin_k != 0))

1
2

df.duplicated(subset=None, keep='first')
f.drop_duplicates(subset=None, keep='first', inplace=False)

1
2
3
4
5

pd.cut( x, bins, right=True, labels=None, retbins=False, precision=3, include_lowest=False, duplicates='raise', )

df['AgeBand'] = pd.cut(df['Age'], 5, labels = ['1','2','3','4','5'])
df['AgeBand'] = pd.cut(df['Age'], [0,5,15,30,50,80], labels = ['1','2','3','4','5'])
df['AgeBand'] = pd.qcut(df['Age'],[0,0.1,0.3,0.5,0.7,0.9],labels = ['1','2','3','4','5'])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

df.replace(to_replace=None, value=None, inplace=False, limit=None, regex=False, method='pad', axis=None)

#LabelEncoder 例子
from sklearn.preprocessing import LabelEncoder
for feat in ['Cabin', 'Ticket']:
    lbl = LabelEncoder()  
    label_dict = dict(zip(df[feat].unique(), range(df[feat].nunique())))
    df[feat + "_labelEncode"] = df[feat].map(label_dict)
    df[feat + "_labelEncode"] = lbl.fit_transform(df[feat].astype(str))

df.head()

1

pandas.get_dummies(data, prefix=None, prefix_sep='_', dummy_na=False, columns=None, sparse=False, drop_first=False, dtype=None)

1
2

pd.concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False, keys=None, levels=None, names=None, verify_integrity=False, copy=True)

1

Series.str.extract(pat, flags=0, expand=None)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

#泰坦尼克号 groupby案例  #统计泰坦尼克号中男女存活人数
test4 = df.groupby(['Sex','Survived']).count()
test4.reset_index(['Survived'], drop = False, inplace = True)
test5 = test4[test4['Survived'] == 1]
test9 = test5.loc[:,['Sex','PassengerId']]
test9.head()


import matplotlib.pyplot as plt
all_male = df['Sex'].value_counts()['male']
all_female = df['Sex'].value_counts()['female']
sur_male = test9.iloc[0,1]
sur_female = test9.iloc[1,1]
male_rate = sur_male / all_male
female_rate = sur_female / all_female
plt.bar(['male','female'],[male_rate, female_rate])
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

test8 = df.groupby(['Embarked','Survived']).count()
test8.reset_index(['Embarked','Survived'], drop = False, inplace = True)
rate =[]
for i in ['C','Q','S']:
    all_ = (test8[test8.Embarked == i]['PassengerId'].sum())
    sur = ((test7.loc[:,['PassengerId']]).loc[i,'PassengerId'])
    rate.append(sur / all_)

plt.bar(['C','Q','S'],rate)
plt.show()

1
2
3
4

df.groupby('Survived').agg({'Sex': 'mean', 'Pclass': 'count'}).rename(columns=
                            {'Sex': 'mean_sex', 'Pclass': 'count_pclass'})

DataFrame.agg（func，axis = 0，* args，** kwargs ）

1
2
3
4
5
6
7
8
9

import numpy as np
from matplotlib import pyplot as plt

y=np.arange(1,5)
plt.plot(y,color='m')
plt.plot(y+1,color='#CFA02F')
plt.plot(y+2,color=(0.2,0.4,0.5))
plt.plot(y+3,color='0.8')  # 0.8代表灰色的深度
plt.show()

1
2
3
4
5
6
7
8
9

import numpy as np
from matplotlib import pyplot as plt

y=np.arange(1,5)
plt.plot(y,'^')
plt.plot(y+1,'--')
plt.plot(y+1.5,'-.')
plt.plot(y+2,':')
plt.show()

1
2
3
4
5
6
7

import numpy as np
from matplotlib import pyplot as plt

X=np.random.randint(10,10000,1000)
Y=np.random.randint(10,10000,1000)
plt.scatter(X,Y,s=20,c='r',alpha=0.5)  #散点图,s是点的大小（面积），默认是20,c是颜色,alpha是透明度
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

import numpy as np 
import matplotlib.pyplot as plt 
# 计算正弦和余弦曲线上的点的 x 和 y 坐标 
x = np.arange(-np.pi,  np.pi,  0.1) 
y_sin = np.sin(x) 
y_cos = np.cos(x)  
# 建立 subplot 网格，高为 2，宽为 1  
# 激活第一个 subplot
plt.subplot(2,  1,  1)  
# 绘制第一个图像 
plt.plot(x, y_sin) 
plt.title('Sine')  
# 将第二个 subplot 激活，并绘制第二个图像
plt.subplot(2,  1,  2) 
plt.plot(x, y_cos) 
plt.title('Cosine')  
# 展示图像
plt.show()

1
2
3
4
5
6
7

import numpy as np
from matplotlib import pyplot as plt

X=np.arange(0,10,0.1)
Y=np.random.randint(0,10000,size=(100,))
plt.plot(X,Y,c='r',alpha=0.5)  
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

import numpy as np
from matplotlib import pyplot as plt

N=5
y=[20,10,30,25,15]
index=np.arange(N)
'''
left为x轴的位置序列(index)
height为y轴的数值序列，也就是柱形图的高度(y)
width为柱形图的宽度
'''
plt.bar(index,y,color='red',width=0.5)
plt.show()

1
2
3
4
5
6
7
8

import numpy as np
from matplotlib import pyplot as plt

N=5
y=[20,10,30,25,15]
index=np.arange(N)
plt.barh(index,y,color='green',height=0.8)
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

import numpy as np
from matplotlib import pyplot as plt

N=5
y=[20,10,30,25,15]
y2=[14,30,22,18,25]
index=np.arange(N)
'''
left为x轴的位置序列(index)
height为y轴的数值序列，也就是柱形图的高度(y)
width为柱形图的宽度
'''
plt.bar(index,y,color='red',width=0.3)
plt.bar(index+0.3,y2,color='yellow',width=0.3)
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

import numpy as np
from matplotlib import pyplot as plt

N=5
y=[20,10,30,25,15]
y2=[14,30,22,18,25]
index=np.arange(N)
'''
left为x轴的位置序列(index)
height为y轴的数值序列，也就是柱形图的高度(y)
width为柱形图的宽度
'''
plt.bar(index,y,color='red',width=0.3)
plt.bar(index,y2,color='yellow',width=0.3,bottom=y)
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

import numpy as np
from matplotlib import pyplot as plt

average=100
devation=20
X=average+devation*np.random.randn(1000)  #在服从标准正态分布的数据中取1000个随机数
'''
画直方图函数的参数：
bins:长方形的数目
normed :为True是频率图，False是频数图，默认为False
'''
plt.hist(X,bins=10,color='red',normed=True)
plt.show()

1
2
3
4
5
6
7

import numpy as np
from matplotlib import pyplot as plt

x=np.random.randn(1000)+2
y=np.random.randn(1000)+5
plt.hist2d(x,y,bins=30)
plt.show()

1
2
3
4
5
6
7
8
9

import matplotlib.pyplot as plt

label=['A','B','C','D']    #标签
fracs=[15,30,45,10]        #比例
plt.axes(aspect=1)
explode=[0,0.05,0.08,0]  #可以理解为扇形圆心到整个圆的圆心之间的距离
plt.pie(x=fracs,labels=label,autopct='%.0f%%',explode=explode,shadow=True)      
#autopct确定输出格式 ,%%表示一个'%'
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

import matplotlib.pyplot as plt
import numpy as np
labels=['A','B','C','D','E']
np.random.seed(1)
data=np.random.normal(size=(1000,5))     #默认随机数服从正态分布（loc=0,scale=1)
'''
sym 调整好异常值的点的形状
whis 默认是1.5， 通过调整数值来设置异常值显示的数量，
如果想显示尽可能多的异常值，whis设置很小，否则很大
'''
plt.boxplot(data,labels=labels,sym='o',whis=1.25)
plt.show()

1

import seaborn as sns

1
2
3
4
5
6
7
8
9

import seaborn as sns
#新建facet 类对象   df：数据  col：值有几种，就生成几个图  hue：类别  aspect：横纵比
facet = sns.FacetGrid(df, col= 'Sex',hue="Survived",aspect=3)
#sns.kdeplot:核密度估计图 'Age':横坐标 'shade':线下方阴影
facet.map(sns.kdeplot,'Age',shade=True)
# xlim x区间
facet.set(xlim=(0, df['Age'].max()))
# 显示
facet.add_legend()

1
2
3
4

test = df.groupby(['Age','Embarked']).count()
test.reset_index(['Age','Embarked'], drop = False , inplace = True)
test1 = test.loc[:,['Age','Embarked','PassengerId']]
sns.lineplot(x='Age', y='PassengerId' ,hue = 'Embarked', data=test1)

1
2
3
4

from sklearn.model_selection import train_test_split
x = clear_df
y = df['Survived']
X_train,X_test, y_train, y_test =train_test_split(x,y,stratify=y, random_state=0)

1

train_test_split (*arrays，test_size, train_size, rondom_state=None, shuffle=True, stratify=None)

1
2
3
4
5

LogisticRegression(C=100, class_weight=None, dual=False, fit_intercept=True,
                   intercept_scaling=1, l1_ratio=None, max_iter=100,
                   multi_class='auto', n_jobs=None, penalty='l2',
                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,
                   warm_start=False)

1
2
3
4

from sklearn.linear_model import LogisticRegression
model = LogisticRegression(penalty='l2',solver='liblinear')
model.fit(X_train, y_train)
model.score(X_test, y_test)

1

DecisionTreeClassifier(*, criterion='gini', splitter='best', max_depth=None, 							min_samples_split=2, min_samples_leaf=1, 										min_weight_fraction_leaf=0.0, max_features=None, 								random_state=None, max_leaf_nodes=None, 										min_impurity_decrease=0.0, min_impurity_split=None, 							class_weight=None, presort='deprecated', ccp_alpha=0.0)

1
2
3
4
5
6
7
8

RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,
                       criterion='gini', max_depth=None, max_features='auto',
                       max_leaf_nodes=None, max_samples=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, n_estimators=100,
                       n_jobs=None, oob_score=False, random_state=None,
                       verbose=0, warm_start=False)

1
2
3
4

from sklearn.model_selection import cross_val_score
lr = LogisticRegression(C=100,solver='liblinear')
scores = cross_val_score(lr, X_train, y_train, cv=10)
print("Average cross-validation score: {:.2f}".format(scores.mean()))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

from sklearn.metrics import confusion_matrix
#训练模型
lr = LogisticRegression(C=100,solver='liblinear')
lr.fit(X_train, y_train)
pred = lr.predict(X_train)
#构建混淆矩阵
confusion_matrix(y_train, pred)

from sklearn.metrics import classification_report
#精确率、召回率以及f1-score
print(classification_report(y_train, pred))