数据准备技巧及其对机器学习的重要性

# 读取数据df_car = pd.read_csv('CarPrice_Assignment.csv')df_car.head(10)  # 显示前十行数据df_car.drop(['car_ID'], axis=1, inplace=True)  # 重复信息，丢弃car_ID列 # 检查列是否包含空数值df_car.isnull().sum()# 丢弃重复行df_car.drop_duplicates(inplace=True)# 根据其类型分配列名cat_variable = []num_variable = []for col in df_car.columns:    if df_car[col].dtype == 'object':        cat_variable.append(col)    elif df_car[col].dtype == 'float' or df_car[col].dtype == 'int64':        num_variable.append(col)        # CarName列中类别数量 df_car['CarName'].value_counts()# 检测数字变量中的离群点df_car[num_variable].describe()

# 分离汽车及其公司名称，从而进行特征分割# 检查Company_name列是否有重复数据 df_car['Company_name'] = df_car['CarName'].apply(lambda x:x.split(' ')[0])df_car['Company_name'].value_counts()df_car.drop(labels='CarName',axis=1,inplace=True)# 创建字典合并相似公司名称company_name_map_dict = {    'maxda' : 'mazda',    'vw' : 'volkswagen',    'toyouta' : 'toyota',    'Nissan' : 'nissan',    'vokswagen' : 'volkswagen',    'porcshce' : 'porsche'}df_car['Company_name'] = df_car['Company_name'].replace(company_name_map_dict)df_car['Company_name'].value_counts()# 绘图显示price与Company_name的关系plt.figure(figsize=(20, 8))sns.boxplot(x = 'Company_name', y = 'price', data = df_car)# 绘图显示数字列是否有线性相关plt.figure(figsize=(30, 25))for i, col in enumerate(num_variable):    plt.subplot(5,3,i+1)    sns.regplot(col, 'price', df_car, fit_reg=False)

    # 将类别列转换为虚拟变量，方便后续ML模型处理  df_car['fueltype'] = df_car['fueltype'].map({'gas':0, 'diesel':1})  df_car['aspiration'] = df_car['aspiration'].map({'std':0, 'turbo':1})  df_car['doornumber'] = df_car['doornumber'].map({'four':0, 'two':1})  df_car['enginelocation'] = df_car['enginelocation'].map({'front':0, 'rear':1})    # 为拥有超过两种类别的列创建虚拟变量  for col in ['carbody', 'drivewheel', 'enginetype', 'cylindernumber', 'fuelsystem', 'Company_name']:      df_car = pd.get_dummies(df_car, columns=[col], drop_first=True)       # 显示将所有类型数据列转换为0-1后的新数据   df_car.head()    # 将数据集分为训练与测试两部分   df_train, df_test = train_test_split(df_car, test_size = 0.15, random_state=44)  # 将所有数字类别列转化为0或1   from sklearn.preprocessing import MinMaxScaler  scaler = MinMaxScaler()  df_train[list(df_train.columns)] = scaler.fit_transform(df_train)  # 取price列为因变量‘y’，其余数据为自变量‘x’   y_train = df_train.pop('price')  X_train = df_train

* 特征选择：用于对数据集中现有特征的重要性进行排序，并剔除不重要的特征。* 特征提取：用于将数据投影到一个较低维度的空间，但依然保留原始数据最重要的特征

def corr_heatmap(df, v):      correlations = df[v].corr()      # Create color map ranging between two colors      cmap = sns.diverging_palette(220, 10, as_cmap=True)      fig, ax = plt.subplots(figsize=(20,20))      sns.heatmap(correlations, cmap=cmap, vmax=1.0, center=0, fmt='.2f',                  square=True, linewidths=.5, annot=True, cbar_kws={"shrink": .75})      plt.show()        # 可视化相关矩阵以排除因变量   corr_heatmap(df_car, num_variable[:-1])    # 丢弃高度相关的属性  df_car = df_car.drop(['citympg', 'curbweight', 'wheelbase', 'enginesize', 'carwidth'], axis=1)  # Importing RFE and LinearRegression  from sklearn.feature_selection import RFE  from sklearn.linear_model import LinearRegression    # Running RFE with the output number of the variable equal to 10  lm = LinearRegression()  lm.fit(X_train, y_train)               # Model_1  rfe = RFE(lm, 10)                      # running RFE  rfe = rfe.fit(X_train, y_train)  # Displaying the list of column with there rfe rank  list(zip(X_train.columns,rfe.support_,rfe.ranking_))  # taking a subset of column, that rfe has suggested important  col = X_train.columns[rfe.support_]    # Creating X_test dataframe with RFE selected variables  X_rfe = X_train[col]  lm.fit(X_rfe, y_train)              # Model_2